Variables and templating (#5)

* Fix issue with using variables in for loops

* Fix critical bug: mutable default value in attrs class

* Substitute in vars in recipe arguments

* Mark failed recipes on the dot output

* Add support for Jinja2 templating

* Antilint

scone/default/steps/fridge_steps.py

@@ -2,7 +2,7 @@ from enum import Enum
 from pathlib import Path, PurePath
 from typing import List, Optional, Tuple, Union
 
-from jinja2 import Template
+from jinja2 import DictLoader, Environment
 
 from scone.head.head import Head
 from scone.head.kitchen import Kitchen
@@ -77,10 +77,24 @@ async def load_and_transform(
         data = head.secret_access.decrypt_bytes(data)
     elif meta == FridgeMetadata.TEMPLATE:
         # pass through Jinja2
-        template = Template(data.decode())
+        try:
-        proxies = kitchen.get_dependency_tracker().get_j2_compatible_dep_var_proxies(
+            env = Environment(
-            head.variables[sous]
+                loader=DictLoader({str(fullpath): data.decode()}), autoescape=False
-        )
+            )
-        data = template.render(proxies).encode()
+            template = env.get_template(str(fullpath))
+            proxies = kitchen.get_dependency_tracker().get_j2_var_proxies(
+                head.variables[sous]
+            )
+            data = template.render(proxies).encode()
+        except Exception as e:
+            raise RuntimeError(f"Error templating: {fullpath}") from e
+
+        # try:
+        #     return jinja2.utils.concat(
+        #         template.root_render_func(template.new_context(proxies))
+        #     )
+        # except Exception:
+        #     template.environment.handle_exception()
+
     print("data", fullpath, data)
     return data

scone/head/dag.py

@@ -30,6 +30,9 @@ class RecipeState(Enum):
     # This recipe has not been cooked because it didn't need to be.
     SKIPPED = 10
 
+    # This recipe failed.
+    FAILED = -1
+
     @staticmethod
     def is_completed(state):
         return state in (RecipeState.COOKED, RecipeState.SKIPPED)

scone/head/dependency_tracking.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import time
+from copy import deepcopy
 from hashlib import sha256
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
@@ -12,6 +13,7 @@ from aiosqlite import Connection
 
 from scone.head.dag import Resource
 from scone.head.recipe import recipe_name_getter
+from scone.head.variables import Variables
 
 if TYPE_CHECKING:
     from scone.head.dag import RecipeDag
@@ -118,7 +120,8 @@ class DependencyTracker:
 
     def register_variable(self, variable: str, value: Union[dict, str, int]):
         # self._vars[variable] = value
-        raise NotImplementedError("time")
+        # TODO(implement)
+        logger.critical("not implemented: register var %s", variable)
 
     def register_fridge_file(self, desugared_path: str):
         # TODO this is not complete
@@ -131,19 +134,48 @@ class DependencyTracker:
         file_res = Resource("file", path, sous=sous)
         self.watch(file_res)
 
-    # def get_j2_compatible_dep_var_proxies(
+    def get_j2_var_proxies(
-    #     self, variables: Variables
+        self, variables: Variables
-    # ) -> Dict[str, "DependencyVarProxy"]:
+    ) -> Dict[str, "DependencyVarProxy"]:
-    #     # XXX BROKEN does not work for overrides
+        result = {}
-    #     result = {}
+
-    #
+        for key in variables.toplevel():
-    #     if len("1"):
+            result[key] = DependencyVarProxy(key, variables, self)
-    #         raise NotImplementedError("BROKEN")
+
-    #
+        return result
-    #     for key, vars in variables.toplevel().items():
+
-    #         result[key] = DependencyVarProxy(self, vars, key + ".")
+
-    #
+class DependencyVarProxy:
-    #     return result
+    def __init__(
+        self,
+        current_path_prefix: Optional[str],
+        vars: Variables,
+        tracker: DependencyTracker,
+    ):
+        self._current_path_prefix = current_path_prefix
+        self._vars = vars
+        self._tracker = tracker
+
+    def raw_(self) -> Dict[str, Any]:
+        if not self._current_path_prefix:
+            raw_dict = self._vars.toplevel()
+        else:
+            raw_dict = self._vars.get_dotted(self._current_path_prefix)
+        self._tracker.register_variable(self._current_path_prefix or "", raw_dict)
+        return deepcopy(raw_dict)
+
+    def __getattr__(self, name: str) -> Union["DependencyVarProxy", Any]:
+        if not self._current_path_prefix:
+            dotted_path = name
+        else:
+            dotted_path = f"{self._current_path_prefix}.{name}"
+        raw_value = self._vars.get_dotted(dotted_path)
+
+        if isinstance(raw_value, dict):
+            return DependencyVarProxy(dotted_path, self._vars, self._tracker)
+        else:
+            self._tracker.register_variable(dotted_path, raw_value)
+            return raw_value
 
 
 class DependencyCache:

scone/head/dot_emitter.py

@@ -5,13 +5,14 @@ from scone.head.dag import RecipeDag, RecipeState, Resource, Vertex
 from scone.head.recipe import Recipe, recipe_name_getter
 
 state_to_colour = {
-    RecipeState.LOADED: "#000000",
+    RecipeState.LOADED: ("white", "black"),
-    RecipeState.PREPARED: "azure",
+    RecipeState.PREPARED: ("azure", "black"),
-    RecipeState.PENDING: "pink",
+    RecipeState.PENDING: ("pink", "black"),
-    RecipeState.COOKABLE: "gold",
+    RecipeState.COOKABLE: ("gold", "black"),
-    RecipeState.COOKED: "darkolivegreen1",
+    RecipeState.COOKED: ("darkolivegreen1", "black"),
-    RecipeState.SKIPPED: "cadetblue1",
+    RecipeState.SKIPPED: ("cadetblue1", "black"),
-    RecipeState.BEING_COOKED: "darkorange1",
+    RecipeState.BEING_COOKED: ("darkorange1", "black"),
+    RecipeState.FAILED: ("black", "orange"),
 }
 
 
@@ -32,10 +33,10 @@ def emit_dot(dag: RecipeDag, path_out: Path) -> None:
                     f"{recipe_name_getter(vertex.__class__)}"
                     f" [{rec_meta.incoming_uncompleted}]"
                 )
-                colour = state_to_colour[rec_meta.state]
+                colour, text_colour = state_to_colour[rec_meta.state]
                 fout.write(
                     f'\t{vertex_id} [shape=box, label="{label}",'
-                    f" style=filled, fillcolor={colour}];\n"
+                    f" style=filled, fontcolor={text_colour}, fillcolor={colour}];\n"
                 )
             elif isinstance(vertex, Resource):
                 label = str(vertex).replace("\\", "\\\\").replace('"', '\\"')

scone/head/kitchen.py

@@ -208,7 +208,11 @@ class Kitchen:
                 self._dependency_trackers[next_job] = DependencyTracker(
                     DependencyBook(), dag, next_job
                 )
-                await next_job.cook(self)
+                try:
+                    await next_job.cook(self)
+                except Exception as e:
+                    meta.state = RecipeState.FAILED
+                    raise RuntimeError(f"Recipe {next_job} failed!") from e
                 eprint(f"cooked {next_job}")
                 # TODO cook
                 # TODO store depbook

scone/head/menu_reader.py

@@ -10,11 +10,11 @@ import textx
 
 from scone.head.dag import RecipeDag, Resource
 from scone.head.recipe import RecipeContext
+from scone.head.variables import Variables
 
 if typing.TYPE_CHECKING:
     from scone.head.head import Head
     from scone.head.recipe import Recipe
-    from scone.head.variables import Variables
 
 
 def _load_grammar():
@@ -69,10 +69,10 @@ class MenuBlock:
 
     user_directive: Optional[str] = None
     sous_directive: Optional[str] = None
-    for_directives: List[ForDirective] = []
+    for_directives: List[ForDirective] = attr.ib(factory=list)
-    import_directives: List[str] = []
+    import_directives: List[str] = attr.ib(factory=list)
-    recipe_edges: List[RecipeEdgeDirective] = []
+    recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list)
-    resource_edges: List[ResourceEdgeDirective] = []
+    resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list)
 
 
 @attr.s(auto_attribs=True, eq=False)
@@ -89,9 +89,9 @@ class MenuRecipe:
 
     user_directive: Optional[str] = None
     sous_directive: Optional[str] = None
-    for_directives: List[ForDirective] = []
+    for_directives: List[ForDirective] = attr.ib(factory=list)
-    recipe_edges: List[RecipeEdgeDirective] = []
+    recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list)
-    resource_edges: List[ResourceEdgeDirective] = []
+    resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list)
 
 
 def convert_textx_value(txvalue) -> Any:
@@ -336,11 +336,13 @@ class MenuLoader:
                     hierarchical_source=hierarchical_source,  # XXX
                     human=recipe.human,
                 )
-                args = recipe.arguments  # noqa
+                try:
-                # XXX sub in vars
+                    args = _vars.substitute_in_dict_copy(recipe.arguments)
-                instance: Recipe = recipe_class.new(
+                except KeyError as ke:
-                    context, recipe.arguments, self._head
+                    raise KeyError(
-                )
+                        f"When substituting for {hierarchical_source} / {recipe}"
+                    ) from ke
+                instance: Recipe = recipe_class.new(context, args, self._head)
                 self._recipes[recipe][(sous, for_indices)] = instance
                 self._dag.add(instance)
 

scone/head/variables.py

@@ -1,5 +1,6 @@
+from copy import deepcopy
 from enum import Enum
-from typing import Any, Dict, List, NamedTuple, Optional
+from typing import Any, Dict, List, NamedTuple, Optional, Set
 
 ExpressionPart = NamedTuple("ExpressionPart", [("kind", str), ("value", str)])
 
@@ -131,14 +132,14 @@ class Variables:
                 self.set_dotted(var_name, sub_val)
                 return sub_val
             else:
-                raise KeyError(f"No variable '{incoming}'")
+                raise KeyError(f"No variable '{var_name}'")
 
         out = ""
         for part in parsed:
             if part.kind == "literal":
                 out += part.value
             elif part.kind == "variable":
-                var_name = parsed[0].value
+                var_name = part.value
                 if self.has_dotted(var_name):
                     out += str(self.get_dotted(var_name))
                 elif var_name in incoming:
@@ -147,7 +148,7 @@ class Variables:
                     self.set_dotted(var_name, sub_val)
                     out += str(sub_val)
                 else:
-                    raise KeyError(f"No variable '{incoming}'")
+                    raise KeyError(f"No variable '{var_name}'")
         return out
 
     def load_vars_with_substitutions(self, incoming: Dict[str, Any]):
@@ -170,5 +171,16 @@ class Variables:
             elif isinstance(v, str):
                 dictionary[k] = self.eval(v)
 
+    def substitute_in_dict_copy(self, dictionary: Dict[str, Any]):
+        new_dict = deepcopy(dictionary)
+        self.substitute_inplace_in_dict(new_dict)
+        return new_dict
+
     def toplevel(self):
         return self._vars
+
+    def keys(self) -> Set[str]:
+        keys = set(self._vars.keys())
+        if self._delegate:
+            keys.update(self._delegate.keys())
+        return keys