Add locking to prevent apt locking issues from concurrent apt ops

scone/default/recipes/apt.py

@@ -17,7 +17,10 @@
 
 import asyncio
 import logging
-from typing import List
+import random
+from asyncio import Lock
+from collections import defaultdict
+from typing import List, Tuple, Dict
 
 from scone.default.utensils.basic_utensils import SimpleExec
 from scone.head.head import Head
@@ -114,6 +117,10 @@ logger = logging.getLogger(__name__)
 #                 )
 
 
+# (id of Kitchen, sous name) → Lock
+apt_locks: Dict[Tuple[int, str], Lock] = defaultdict(Lock)
+
+
 class AptPackage(Recipe):
     _NAME = "apt-install"
 
@@ -135,6 +142,8 @@ class AptPackage(Recipe):
         while retries > 0:
             result = await kitchen.ut1areq(SimpleExec(args, "/"), SimpleExec.Result)
 
+            logger.debug("E %r: %r", args, result.stderr)
+
             if result.exit_code == 0 or b"/lock" not in result.stderr:
                 return result
 
@@ -142,8 +151,6 @@ class AptPackage(Recipe):
                 "Failed apt command due to suspected locking issue. Will retry…"
             )
 
-            retries -= 1
-
             # /lock seen in stderr, probably a locking issue...
             lock_check = await kitchen.ut1areq(
                 SimpleExec(
@@ -160,7 +167,8 @@ class AptPackage(Recipe):
                 )
                 retries -= 1
 
-            await asyncio.sleep(2.0)
+            sleep = 2.0 + 3.0 * random.random()
+            await asyncio.sleep(sleep)
 
         return result  # noqa
 
@@ -168,7 +176,13 @@ class AptPackage(Recipe):
         # this is a one-off task assuming everything works
         kitchen.get_dependency_tracker()
 
-        if self.packages:
+        lock = apt_locks[(id(kitchen), self.recipe_context.sous)]
+
+        # we only want one apt task to run at once on each sous because they tend
+        # to race against each other and lock each other
+        async with lock:
+            if not self.packages:
+                return
             update = await self._apt_command(kitchen, ["apt-get", "-yq", "update"])
             if update.exit_code != 0:
                 raise RuntimeError(