Docker enhancements: containers (#22)

Antilint

Document docker work

Provide docker containers as resources

Fix up arguments passed to docker, and remove error suppression

Fix serialisation of enum values by encoding as int

Add more docker-container features.

Fix parsing bug

Update documentation

Fix grammar to accept more dictionaries

Antilint

Add volumes and ports to docker-container recipe

Extend DockerContainerRun with some useful parameters

Antilint

Require name parameter in docker-container recipe

Check for docker support being installed

Add ability to give containers names & check containers by name

Co-authored-by: Olivier <olivier@librepush.net>
Reviewed-on: https://bics.ga/reivilibre/scone/pulls/22
Co-Authored-By: Olivier 'reivilibre' <reivilibre@noreply.%(DOMAIN)s>
Co-Committed-By: Olivier 'reivilibre' <reivilibre@noreply.%(DOMAIN)s>

docs/src/recipes/docker.md

@@ -0,0 +1,52 @@
+# Docker
+
+This is work-in-progress integration with Docker for Scone.
+
+For this set of recipes to work, you will need the sous to have the `docker`
+Python package installed; for this, you can use `pip install docker` with the
+scone virtualenv activated. (TODO link to better documentation about installing
+other Python deps in a scone venv.)
+
+| Recipe | Needs | Provides |
+| -----: | ----- | -------- |
+| [`docker-container`](#docker-container) | | `docker_container` |
+
+## `docker-container`
+
+**Preconditions**: the image must be available or downloadable
+
+**Provides**: `docker-container(?)` where `?` is the argument `name`.
+
+| Argument | Accepted Values | Default | Description |
+| -------: | --------------- | ------- | ----------- |
+| image | any Docker image that exists or can be pulled | *required* | This name is used to specify what image to install. |
+| name | any ID string | *required* | This name identifies the container and is passed to docker. |
+| command | any command string | *optional* | If specified, this sets the command that will be run by the container. |
+| ports | dictionary of "(container port)/tcp" or "(container port)/udp" to {host = "(host address)", port = "(host port)"} | empty | This mapping describes how ports are published from inside the container to the host. |
+| volumes | dictionary of "(volume name)" or "/path/to/binding/on/host" to {bind = "/path/in/container", mode = "rw" or "ro"} | empty | This mapping describes what filesystem resources are mounted into the container. |
+| environment | dictionary of "(key)" to value | empty | This mapping describes what environment variables are given to the container. |
+| restart_policy | "always" or "on-failure" | "on-failure" | This specifies the container's restart policy. |
+
+### Example
+
+```scoml
+[[docker-container]]
+image = "org/image:1"
+name = "mycontainer"
+ports = {
+  "80/tcp" = {
+    "host" = "127.0.0.1",
+    "port" = 4080
+  }
+}
+volumes = {
+  "/var/lib/mycontainer" = {
+    bind = "/data",
+    mode = "rw"
+  }
+}
+environment = {
+  MYCONTAINER_MODE = "production"
+}
+restart_policy = "always"
+```

scone/default/recipes/docker.py

@@ -1,10 +1,13 @@
 from scone.default.utensils.docker_utensils import (
+    ContainerState,
     DockerContainerRun,
+    DockerContainerState,
     DockerImagePull,
     DockerNetworkCreate,
     DockerVolumeCreate,
 )
-from scone.head.kitchen import Kitchen
+from scone.head.head import Head
+from scone.head.kitchen import Kitchen, Preparation
 from scone.head.recipe import Recipe, RecipeContext
 from scone.head.utils import check_type, check_type_opt
 
@@ -16,14 +19,38 @@ class DockerContainer(Recipe):
         super().__init__(recipe_context, args, head)
 
         self.image = check_type(args.get("image"), str)
-        self.command = check_type(args.get("command"), str)
+        self.command = check_type_opt(args.get("command"), str)
+        self.name = check_type(args.get("name"), str)
+        self.volumes = check_type(args.get("volumes", dict()), dict)
+        self.ports = check_type(args.get("ports", dict()), dict)
+        self.environment = check_type(args.get("environment", dict()), dict)
+        self.restart_policy = check_type(args.get("restart_policy", "on-failure"), str)
+
+    def prepare(self, preparation: Preparation, head: Head) -> None:
+        super().prepare(preparation, head)
+        preparation.provides("docker-container", self.name)
 
     async def cook(self, kitchen: Kitchen) -> None:
         kitchen.get_dependency_tracker()
-        await kitchen.ut1areq(
-            DockerContainerRun(self.image, self.command), DockerContainerRun.Result
+
+        current_state = ContainerState(
+            await kitchen.ut1(DockerContainerState(self.name))
         )
 
+        if current_state == ContainerState.NOTFOUND:
+            await kitchen.ut1areq(
+                DockerContainerRun(
+                    self.image,
+                    self.command,
+                    self.name,
+                    {k: (v["host"], v["port"]) for k, v in self.ports.items()},
+                    self.volumes,
+                    {k: str(v) for k, v in self.environment.items()},
+                    self.restart_policy,
+                ),
+                DockerContainerRun.Result,
+            )
+
 
 class DockerImage(Recipe):
     _NAME = "docker-image"

scone/default/steps/filesystem_steps.py

@@ -15,7 +15,6 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Scone.  If not, see <https://www.gnu.org/licenses/>.
 
-from scone.default.utensils.basic_utensils import HashFile
 from scone.head.kitchen import Kitchen
 
 

scone/default/utensils/docker_utensils.py

@@ -1,11 +1,14 @@
-from typing import Optional
+from enum import Enum
+from typing import Dict, Optional, Tuple
 
 import attr
 
 try:
     import docker.errors
+    from docker.models.containers import Container
 except ImportError:
     docker = None
+    Container = None
 
 from scone.common.chanpro import Channel
 from scone.sous import Utensil
@@ -15,35 +18,94 @@ _docker_client_instance = None
 
 
 def _docker_client():
+    if not docker:
+        # check docker is actually installed and give a message with the resolution
+        # when it isn't.
+        raise RuntimeError(
+            "You need to install docker from PyPI to use these utensils!"
+        )
+
     global _docker_client_instance
     if not _docker_client_instance:
         _docker_client_instance = docker.from_env()
     return _docker_client_instance
 
 
+class ContainerState(Enum):
+    NOTFOUND = 0
+    RUNNING = 1
+    EXITED = 2
+    RESTARTING = 3
+
+
+@attr.s(auto_attribs=True)
+class DockerContainerState(Utensil):
+    # Name of the container to check the existence of.
+    name: str
+
+    async def execute(self, channel: Channel, worktop: Worktop):
+        client = _docker_client()
+        # this is essentially `docker ps -a`
+        # TODO(perf) run this in a threaded executor since docker can be slow.
+        for container in client.containers.list(all=True):
+            container: Container
+            if self.name == container.name:
+                if container.status == "running":
+                    await channel.send(ContainerState.RUNNING.value)
+                elif container.status == "exited":
+                    await channel.send(ContainerState.EXITED.value)
+                elif container.status == "restarting":
+                    await channel.send(ContainerState.RESTARTING.value)
+                else:
+                    raise ValueError(f"Unknown container status: {container.status}")
+                break
+        else:
+            await channel.send(ContainerState.NOTFOUND.value)
+
+
 @attr.s(auto_attribs=True)
 class DockerContainerRun(Utensil):
+    # Name of the image to use to create the container.
     image: str
-    command: str
+    # Command to create the container with. Optional.
+    command: Optional[str]
+    # Custom name to give the container.
+    name: str
+    # Ports to bind inside the container
+    # {'2222/tcp': ('127.0.0.1', 3333)} will expose port 2222 inside as 3333 outside.
+    ports: Dict[str, Tuple[str, int]]
+    # Volumes to mount inside the container.
+    # Key is either a host path or a container name.
+    # Value is a dictionary with the keys of:
+    #   bind = path to bind inside the container
+    #   mode = 'rw' or 'ro'
+    volumes: Dict[str, Dict[str, str]]
+    # Environment variables
+    environment: Dict[str, str]
+    # Restart policy
+    restart_policy: str
 
     @attr.s(auto_attribs=True)
     class Result:
         name: str
 
     async def execute(self, channel: Channel, worktop: Worktop):
-        try:
-            container = _docker_client().containers.run(
-                self.image, self.command, detach=True
-            )
+        restart_policy = {
+            "Name": self.restart_policy,
+        }
+        if self.restart_policy == "on-failure":
+            restart_policy["MaximumRetryCount"] = 5
 
-        except docker.errors.ImageNotFound:
-            # specified image does not exist (or requires login)
-            await channel.send(None)
-            return
-        except docker.errors.APIError:
-            # the docker server returned an error
-            await channel.send(None)
-            return
+        container = _docker_client().containers.run(
+            self.image,
+            self.command,
+            detach=True,
+            name=self.name,
+            ports=self.ports,
+            volumes=self.volumes,
+            environment=self.environment,
+            restart_policy=restart_policy,
+        )
 
         await channel.send(DockerContainerRun.Result(name=container.name))
 

scone/head/grammar/scoml.tx

@@ -133,7 +133,7 @@ QuotedString:
 ;
 
 UnquotedString:
-  value=/[^\s\n,"()0-9]([^\n,"()]*[^\s\n,"()])?/
+  value=/[^]\s\n,"(){}[0-9=]([^]\n,"(){}[=]*[^]\s\n,"(){}[=])?/
 ;
 
 DottedIdString:
@@ -158,12 +158,12 @@ BracketList[ws=' \t\n']:
   ']'
 ;
 
-BraceDict[ws=' \t']:
+BraceDict[ws=' \t\n']:
   '{'
   pairs*=DictPair[',']
   '}'
 ;
 
-DictPair:
+DictPair[ws=' \t\n']:
   (key=KeyExpr) '=' (value=ValueExpr)
 ;

scone/head/menu_reader.py

@@ -120,7 +120,7 @@ class MenuRecipe:
     listen_edges: List[ListenEdgeDirective] = attr.ib(factory=list)
 
 
-def convert_textx_value(txvalue) -> Any:
+def convert_textx_value(txvalue) -> Union[list, str, int, bool, dict]:
     if isinstance(txvalue, scoml_classes["NaturalList"]):
         return [convert_textx_value(element) for element in txvalue.elements]
     elif (
@@ -136,6 +136,7 @@ def convert_textx_value(txvalue) -> Any:
         result = dict()
         for pair in txvalue.pairs:
             result[convert_textx_value(pair.key)] = convert_textx_value(pair.value)
+        return result
     else:
         raise ValueError(f"Unknown SCOML value: {txvalue}")