mirror of
https://github.com/apache/cloudstack
synced 2026-08-02 05:26:35 +00:00
[Veeam] Fix for dirty context not being returned on el9
The dirty extents were not ready correctly for the block_status_64 return and as a result image server always reported the whole image range as dirty. This commit fixes by normalizing the value returned by both block_status and block_status_64. It also adds warning whenever there is an issue getting the dirty extents and the code falls back to returning the full range as dirty. Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com>
This commit is contained in:
Nicolas Vazquez
GitHub
parent
66132f83ea
commit
a6b31286f0
@ -32,6 +32,24 @@ from ..util import coalesce_allocation_extents, merge_dirty_zero_extents
|
||||
from .base import BackendSession, ImageBackend
|
||||
|
||||
|
||||
def _entries_to_pairs(entries: Any) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Normalize block status callback entries to (length, flags) pairs.
|
||||
block_status delivers a flat list [length, flags, ...] while
|
||||
block_status_64 delivers a list of (length, flags) tuples.
|
||||
"""
|
||||
raw = list(entries)
|
||||
if not raw:
|
||||
return []
|
||||
if isinstance(raw[0], (tuple, list)):
|
||||
return [(int(length), int(flags)) for length, flags in raw]
|
||||
if isinstance(raw[0], int):
|
||||
return [(int(raw[i]), int(raw[i + 1])) for i in range(0, len(raw) - 1, 2)]
|
||||
raise TypeError(
|
||||
f"unrecognized block status entry type: {type(raw[0]).__name__}"
|
||||
)
|
||||
|
||||
|
||||
class NbdConnection:
|
||||
"""
|
||||
Low-level helper to connect to an NBD server over a Unix socket.
|
||||
@ -190,19 +208,16 @@ class NbdConnection:
|
||||
return 0
|
||||
current = off
|
||||
try:
|
||||
flat = list(entries)
|
||||
for i in range(0, len(flat), 2):
|
||||
if i + 1 >= len(flat):
|
||||
break
|
||||
length = int(flat[i])
|
||||
flags = int(flat[i + 1])
|
||||
for length, flags in _entries_to_pairs(entries):
|
||||
zero = (flags & (NBD_STATE_HOLE | NBD_STATE_ZERO)) != 0
|
||||
allocation_extents.append(
|
||||
{"start": current, "length": length, "zero": zero}
|
||||
)
|
||||
current += length
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
except (TypeError, ValueError, IndexError) as e:
|
||||
logging.warning(
|
||||
"get_allocation_extents: unparseable block status entries: %r", e
|
||||
)
|
||||
return 0
|
||||
|
||||
block_status_fn = getattr(
|
||||
@ -261,12 +276,7 @@ class NbdConnection:
|
||||
return 0
|
||||
current = off
|
||||
try:
|
||||
flat = list(entries)
|
||||
for i in range(0, len(flat), 2):
|
||||
if i + 1 >= len(flat):
|
||||
break
|
||||
length = int(flat[i])
|
||||
flags = int(flat[i + 1])
|
||||
for length, flags in _entries_to_pairs(entries):
|
||||
if metacontext == "base:allocation":
|
||||
zero = (flags & (NBD_STATE_HOLE | NBD_STATE_ZERO)) != 0
|
||||
allocation_extents.append((current, length, zero))
|
||||
@ -274,8 +284,11 @@ class NbdConnection:
|
||||
dirty = (flags & NBD_STATE_DIRTY) != 0
|
||||
dirty_extents.append((current, length, dirty))
|
||||
current += length
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
except (TypeError, ValueError, IndexError) as e:
|
||||
logging.warning(
|
||||
"get_extents_dirty_and_zero: unparseable block status entries: %r",
|
||||
e,
|
||||
)
|
||||
return 0
|
||||
|
||||
block_status_fn = getattr(
|
||||
|
||||
@ -746,6 +746,11 @@ class Handler(BaseHTTPRequestHandler):
|
||||
dirty_bitmap_ctx = f"qemu:dirty-bitmap:{export_bitmap}"
|
||||
extents = backend.get_dirty_extents(dirty_bitmap_ctx)
|
||||
if is_fallback_dirty_response(extents):
|
||||
logging.warning(
|
||||
"EXTENTS image_id=%s: no usable dirty bitmap data for "
|
||||
"context %r; reporting all allocated extents as dirty",
|
||||
image_id, dirty_bitmap_ctx,
|
||||
)
|
||||
allocation = backend.get_allocation_extents()
|
||||
extents = [
|
||||
{
|
||||
|
||||
194
scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_entries.py
Normal file
194
scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_entries.py
Normal file
@ -0,0 +1,194 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Tests for NBD block status entry parsing.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from imageserver.backends.nbd import NbdConnection, _entries_to_pairs
|
||||
from imageserver.util import is_fallback_dirty_response
|
||||
|
||||
class TestRealLibnbdBlockStatus(unittest.TestCase):
|
||||
"""
|
||||
Creates a 1 MiB qcow2 with a persistent dirty bitmap, dirties two
|
||||
64 KiB regions, exports it via qemu-nbd --bitmap, and asserts both the
|
||||
raw callback entry format and the parsed extents. Fails loudly if a
|
||||
new libnbd version changes the block status response.
|
||||
"""
|
||||
|
||||
REAL_BITMAP = "bm0"
|
||||
KB64 = 65536
|
||||
REAL_SIZE = 16 * KB64 # 1 MiB
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._tmp = tempfile.mkdtemp(prefix="nbd_entries_test_")
|
||||
cls._img = os.path.join(cls._tmp, "img.qcow2")
|
||||
cls._sock = os.path.join(cls._tmp, "nbd.sock")
|
||||
cls._proc = None
|
||||
|
||||
def run(*cmd):
|
||||
subprocess.run(cmd, check=True, capture_output=True)
|
||||
|
||||
try:
|
||||
run("qemu-img", "create", "-f", "qcow2", cls._img, str(cls.REAL_SIZE))
|
||||
# allocate 0-192 KiB before the bitmap exists (clean, allocated)
|
||||
run("qemu-io", "-f", "qcow2", "-c", "write -P 0xaa 0 192k", cls._img)
|
||||
run("qemu-img", "bitmap", "--add", cls._img, cls.REAL_BITMAP)
|
||||
# dirty two 64 KiB regions while the bitmap is recording
|
||||
run("qemu-io", "-f", "qcow2", "-c", "write -P 0xbb 128k 64k", cls._img)
|
||||
run("qemu-io", "-f", "qcow2", "-c", "write -P 0xcc 512k 64k", cls._img)
|
||||
|
||||
cls._proc = subprocess.Popen(
|
||||
[
|
||||
"qemu-nbd",
|
||||
"--socket", cls._sock,
|
||||
"--format", "qcow2",
|
||||
"--persistent",
|
||||
"--shared=0",
|
||||
"--read-only",
|
||||
f"--bitmap={cls.REAL_BITMAP}",
|
||||
cls._img,
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
from .test_base import _wait_for_nbd_socket
|
||||
|
||||
_wait_for_nbd_socket(cls._sock)
|
||||
except BaseException:
|
||||
cls._teardown()
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls._teardown()
|
||||
|
||||
@classmethod
|
||||
def _teardown(cls):
|
||||
if cls._proc is not None:
|
||||
for pipe in (cls._proc.stdout, cls._proc.stderr):
|
||||
if pipe:
|
||||
pipe.close()
|
||||
cls._proc.terminate()
|
||||
try:
|
||||
cls._proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls._proc.kill()
|
||||
cls._proc.wait(timeout=5)
|
||||
cls._proc = None
|
||||
shutil.rmtree(cls._tmp, ignore_errors=True)
|
||||
|
||||
def _connect(self):
|
||||
return NbdConnection(
|
||||
self._sock,
|
||||
None,
|
||||
need_block_status=True,
|
||||
extra_meta_contexts=[f"qemu:dirty-bitmap:{self.REAL_BITMAP}"],
|
||||
)
|
||||
|
||||
def test_raw_entries_format_is_known(self):
|
||||
"""The raw callback entries must be flat ints or (length, flags) pairs."""
|
||||
with self._connect() as conn:
|
||||
size = conn.size()
|
||||
captured = []
|
||||
|
||||
def cb(*args, **kwargs):
|
||||
self.assertGreaterEqual(len(args), 3)
|
||||
captured.append((args[0], list(args[2])))
|
||||
return 0
|
||||
|
||||
fn = getattr(
|
||||
conn._nbd,
|
||||
"block_status_64",
|
||||
getattr(conn._nbd, "block_status", None),
|
||||
)
|
||||
self.assertIsNotNone(
|
||||
fn, "libnbd no longer exposes block_status/block_status_64"
|
||||
)
|
||||
fn(size, 0, cb)
|
||||
|
||||
self.assertTrue(captured, "block status delivered no callbacks")
|
||||
contexts = {ctx for ctx, _ in captured}
|
||||
self.assertIn("base:allocation", contexts)
|
||||
self.assertIn(f"qemu:dirty-bitmap:{self.REAL_BITMAP}", contexts)
|
||||
|
||||
for ctx, entries in captured:
|
||||
self.assertTrue(entries, f"empty entries for context {ctx}")
|
||||
if isinstance(entries[0], (tuple, list)):
|
||||
for item in entries:
|
||||
self.assertIsInstance(item, (tuple, list))
|
||||
self.assertEqual(len(item), 2)
|
||||
self.assertIsInstance(item[0], int)
|
||||
self.assertIsInstance(item[1], int)
|
||||
else:
|
||||
self.assertTrue(all(isinstance(v, int) for v in entries))
|
||||
self.assertEqual(len(entries) % 2, 0)
|
||||
# entries must parse and tile part of the queried range
|
||||
pairs = _entries_to_pairs(entries)
|
||||
self.assertTrue(pairs)
|
||||
covered = sum(length for length, _ in pairs)
|
||||
self.assertGreater(covered, 0)
|
||||
self.assertLessEqual(covered, size)
|
||||
|
||||
def test_allocation_extents_tile_image(self):
|
||||
with self._connect() as conn:
|
||||
size = conn.size()
|
||||
extents = conn.get_allocation_extents()
|
||||
self.assertEqual(size, self.REAL_SIZE)
|
||||
pos = 0
|
||||
for e in extents:
|
||||
self.assertEqual(e["start"], pos)
|
||||
self.assertGreater(e["length"], 0)
|
||||
pos += e["length"]
|
||||
self.assertEqual(pos, size)
|
||||
self.assertEqual(
|
||||
extents,
|
||||
[
|
||||
{"start": 0, "length": 3 * self.KB64, "zero": False},
|
||||
{"start": 3 * self.KB64, "length": 5 * self.KB64, "zero": True},
|
||||
{"start": 8 * self.KB64, "length": self.KB64, "zero": False},
|
||||
{"start": 9 * self.KB64, "length": 7 * self.KB64, "zero": True},
|
||||
],
|
||||
)
|
||||
|
||||
def test_dirty_extents_match_writes(self):
|
||||
with self._connect() as conn:
|
||||
extents = conn.get_extents_dirty_and_zero(
|
||||
f"qemu:dirty-bitmap:{self.REAL_BITMAP}"
|
||||
)
|
||||
self.assertEqual(
|
||||
extents,
|
||||
[
|
||||
{"start": 0, "length": 2 * self.KB64, "dirty": False, "zero": False},
|
||||
{"start": 2 * self.KB64, "length": self.KB64, "dirty": True, "zero": False},
|
||||
{"start": 3 * self.KB64, "length": 5 * self.KB64, "dirty": False, "zero": True},
|
||||
{"start": 8 * self.KB64, "length": self.KB64, "dirty": True, "zero": False},
|
||||
{"start": 9 * self.KB64, "length": 7 * self.KB64, "dirty": False, "zero": True},
|
||||
],
|
||||
)
|
||||
self.assertFalse(is_fallback_dirty_response(extents))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user
