Add more check/GC tests

testsuite/helpers/__init__.py

@@ -172,3 +172,28 @@ def scan_dir(path: Path) -> DirectoryDescriptor:
         file_stat.st_uid,
         file_stat.st_gid,
     )
+
+
+def randomly_mutate_file_in_descriptor(
+    descriptor: FileDescriptor, path: Path, random: Random
+) -> None:
+    with path.open("r+b") as file:
+        length_of_file = path.stat().st_size
+        mutations_to_make = random.randint(1, 3)
+        for _ in range(mutations_to_make):
+            mutate_at_position = random.randint(0, length_of_file - 4)
+            replace_with = random.randbytes(4)
+            file.seek(mutate_at_position)
+            file.write(replace_with)
+
+
+def randomly_mutate_directory_in_descriptor(
+    descriptor: DirectoryDescriptor, path: Path, random: Random
+) -> None:
+    for name, value in descriptor.contents.items():
+        if isinstance(value, FileDescriptor):
+            if random.random() < 0.6:
+                randomly_mutate_file_in_descriptor(value, path.joinpath(name), random)
+        else:
+            assert isinstance(value, DirectoryDescriptor)
+            randomly_mutate_directory_in_descriptor(value, path.joinpath(name), random)

testsuite/yamatests/test_check_and_gc.py

@@ -99,3 +99,205 @@ class TestYamaCheck(TestCase):
         self.assertEqual(ec_deep, 1)
 
         td.cleanup()
+
+    def test_check_succeeds_after_full_removal(self):
+        td = TemporaryDirectory("test_check_fails_after_random_corruption")
+        tdpath = Path(td.name)
+
+        datman_path = tdpath.joinpath("datman")
+        src_path = datman_path.joinpath("srca")
+        yama_path = datman_path.joinpath("main")
+
+        set_up_simple_datman(datman_path)
+        set_up_simple_yama(yama_path)
+
+        rng = Random()
+        seed = rng.randint(0, 9001)
+        print(f"seed: {seed}")
+        rng.seed(seed)
+        later_expected_descriptor, _ = generate_random_dir(rng, src_path, 32)
+
+        subprocess.check_call(("datman", "backup-one", "srca", "main"), cwd=datman_path)
+
+        subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+        subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+        # Find the pointer names and remove the latest one
+        pointer_name = (
+            subprocess.check_output(("yama", "debug", "lsp"), cwd=yama_path)
+            .decode()
+            .split("\n")[0]
+        )
+        subprocess.check_call(("yama", "debug", "rmp", pointer_name), cwd=yama_path)
+
+        # The repository should still be safe.
+        ec_shallow = subprocess.Popen(
+            ("yama", "check", "--shallow"), cwd=yama_path
+        ).wait()
+        ec_deep = subprocess.Popen(("yama", "check", "--deep"), cwd=yama_path).wait()
+        self.assertEqual(ec_shallow, 0)
+        self.assertEqual(ec_deep, 0)
+
+    def _test_gc_safely_clears_space_after_removal(
+        self, depth: str, is_full: bool, which_to_remove: Optional[int] = None
+    ):
+        td = TemporaryDirectory("test_check_fails_after_random_corruption")
+        tdpath = Path(td.name)
+
+        datman_path = tdpath.joinpath("datman")
+        src_path = datman_path.joinpath("srca")
+        yama_path = datman_path.joinpath("main")
+
+        set_up_simple_datman(datman_path)
+        set_up_simple_yama(yama_path)
+
+        rng = Random()
+        seed = rng.randint(0, 9001)
+        print(f"seed: {seed}")
+        rng.seed(seed)
+        later_expected_descriptor, _ = generate_random_dir(rng, src_path, 32)
+
+        subprocess.check_call(("datman", "backup-one", "srca", "main"), cwd=datman_path)
+
+        subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+        subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+        # Find the pointer names and remove the latest one
+        orig_pointer_name = (
+            subprocess.check_output(("yama", "debug", "lsp"), cwd=yama_path)
+            .decode()
+            .split("\n")[0]
+        )
+        if is_full:
+            subprocess.check_call(
+                ("yama", "debug", "rmp", orig_pointer_name), cwd=yama_path
+            )
+        else:
+            assert which_to_remove is not None
+            # we want to add a new, incremental, pointer
+            randomly_mutate_directory_in_descriptor(
+                later_expected_descriptor, src_path, rng
+            )
+
+            subprocess.check_call(
+                ("datman", "backup-one", "srca", "main"), cwd=datman_path
+            )
+
+            subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+            subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+            should_be_orig_pointer_name, new_pointer_name = (
+                subprocess.check_output(("yama", "debug", "lsp"), cwd=yama_path)
+                .decode()
+                .split("\n")[0:2]
+            )
+            self.assertEqual(should_be_orig_pointer_name, orig_pointer_name)
+            self.assertNotEqual(new_pointer_name, should_be_orig_pointer_name)
+            self.assertGreater(len(new_pointer_name.strip()), 1)
+            self.assertTrue(new_pointer_name.startswith("srca+"))
+
+            victim_pointer_name = [orig_pointer_name, new_pointer_name][which_to_remove]
+            subprocess.check_call(
+                ("yama", "debug", "rmp", victim_pointer_name), cwd=yama_path
+            )
+
+        subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+        subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+        output = subprocess.check_output(
+            ("yama", "check", f"--{depth}", "--apply-gc"),
+            cwd=yama_path,
+            stderr=subprocess.STDOUT,
+        )
+        self.assertNotIn(b" 0 chunks", output)
+        self.assertIn(b" chunks", output)
+
+        # The repository should still be safe.
+        ec_shallow = subprocess.Popen(
+            ("yama", "check", "--shallow"), cwd=yama_path
+        ).wait()
+        ec_deep = subprocess.Popen(("yama", "check", "--deep"), cwd=yama_path).wait()
+        self.assertEqual(ec_shallow, 0)
+        self.assertEqual(ec_deep, 0)
+
+    def test_deep_gc_safely_clears_space_after_full_removal(self):
+        self._test_gc_safely_clears_space_after_removal("deep", is_full=True)
+
+    def test_shallow_gc_safely_clears_space_after_full_removal(self):
+        self._test_gc_safely_clears_space_after_removal("shallow", is_full=True)
+
+    def test_deep_gc_safely_clears_space_after_incremental_base_removal(self):
+        self._test_gc_safely_clears_space_after_removal(
+            "deep", is_full=False, which_to_remove=0
+        )
+
+    def test_shallow_gc_safely_clears_space_after_incremental_base_removal(self):
+        self._test_gc_safely_clears_space_after_removal(
+            "shallow", is_full=False, which_to_remove=0
+        )
+
+    def test_deep_gc_safely_clears_space_after_incremental_tail_removal(self):
+        self._test_gc_safely_clears_space_after_removal(
+            "deep", is_full=False, which_to_remove=1
+        )
+
+    def test_shallow_gc_safely_clears_space_after_incremental_tail_removal(self):
+        self._test_gc_safely_clears_space_after_removal(
+            "shallow", is_full=False, which_to_remove=1
+        )
+
+    def test_shallow_and_deep_gc_remove_the_same(self):
+        td = TemporaryDirectory("test_check_fails_after_random_corruption")
+        tdpath = Path(td.name)
+
+        datman_path = tdpath.joinpath("datman")
+        src_path = datman_path.joinpath("srca")
+        yama_path = datman_path.joinpath("main")
+
+        set_up_simple_datman(datman_path)
+        set_up_simple_yama(yama_path)
+
+        rng = Random()
+        seed = rng.randint(0, 9001)
+        print(f"seed: {seed}")
+        rng.seed(seed)
+        later_expected_descriptor, _ = generate_random_dir(rng, src_path, 32)
+
+        subprocess.check_call(("datman", "backup-one", "srca", "main"), cwd=datman_path)
+
+        subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+        subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+        # Find the pointer names and remove the latest one
+        pointer_name = (
+            subprocess.check_output(("yama", "debug", "lsp"), cwd=yama_path)
+            .decode()
+            .split("\n")[0]
+        )
+        subprocess.check_call(("yama", "debug", "rmp", pointer_name), cwd=yama_path)
+
+        subprocess.check_call(("yama", "check", "--shallow"), cwd=yama_path)
+        subprocess.check_call(("yama", "check", "--deep"), cwd=yama_path)
+
+        shallow_output = subprocess.check_output(
+            ("yama", "check", "--shallow", "--dry-run-gc"),
+            cwd=yama_path,
+            stderr=subprocess.STDOUT,
+        ).decode()
+        self.assertNotIn(" 0 chunks", shallow_output)
+        self.assertIn(" chunks", shallow_output)
+
+        deep_output = subprocess.check_output(
+            ("yama", "check", "--deep", "--dry-run-gc"),
+            cwd=yama_path,
+            stderr=subprocess.STDOUT,
+        ).decode()
+        self.assertNotIn(" 0 chunks", deep_output)
+        self.assertIn(" chunks", deep_output)
+
+        pat = re.compile(" ([0-9]+) chunks")
+
+        shallow_chunks = int(pat.search(shallow_output).group(1))
+        deep_chunks = int(pat.search(deep_output).group(1))
+
+        self.assertEqual(shallow_chunks, deep_chunks)