From 04241dc47f42b04c8d2ce89038e6322f89388b14 Mon Sep 17 00:00:00 2001 From: reivilibre <38398653+reivilibre@users.noreply.github.com> Date: Fri, 23 Oct 2020 09:32:06 +0100 Subject: [PATCH] Variables and templating (#5) * Fix issue with using variables in for loops * Fix critical bug: mutable default value in attrs class * Substitute in vars in recipe arguments * Mark failed recipes on the dot output * Add support for Jinja2 templating * Antilint --- scone/default/steps/fridge_steps.py | 26 ++++++++++--- scone/head/dag.py | 3 ++ scone/head/dependency_tracking.py | 60 ++++++++++++++++++++++------- scone/head/dot_emitter.py | 19 ++++----- scone/head/kitchen.py | 6 ++- scone/head/menu_reader.py | 28 +++++++------- scone/head/variables.py | 20 ++++++++-- 7 files changed, 115 insertions(+), 47 deletions(-) diff --git a/scone/default/steps/fridge_steps.py b/scone/default/steps/fridge_steps.py index a8d9365..7443246 100644 --- a/scone/default/steps/fridge_steps.py +++ b/scone/default/steps/fridge_steps.py @@ -2,7 +2,7 @@ from enum import Enum from pathlib import Path, PurePath from typing import List, Optional, Tuple, Union -from jinja2 import Template +from jinja2 import DictLoader, Environment from scone.head.head import Head from scone.head.kitchen import Kitchen @@ -77,10 +77,24 @@ async def load_and_transform( data = head.secret_access.decrypt_bytes(data) elif meta == FridgeMetadata.TEMPLATE: # pass through Jinja2 - template = Template(data.decode()) - proxies = kitchen.get_dependency_tracker().get_j2_compatible_dep_var_proxies( - head.variables[sous] - ) - data = template.render(proxies).encode() + try: + env = Environment( + loader=DictLoader({str(fullpath): data.decode()}), autoescape=False + ) + template = env.get_template(str(fullpath)) + proxies = kitchen.get_dependency_tracker().get_j2_var_proxies( + head.variables[sous] + ) + data = template.render(proxies).encode() + except Exception as e: + raise RuntimeError(f"Error templating: {fullpath}") from e + + # try: + # return jinja2.utils.concat( + # template.root_render_func(template.new_context(proxies)) + # ) + # except Exception: + # template.environment.handle_exception() + print("data", fullpath, data) return data diff --git a/scone/head/dag.py b/scone/head/dag.py index 4bed73b..96cd7ad 100644 --- a/scone/head/dag.py +++ b/scone/head/dag.py @@ -30,6 +30,9 @@ class RecipeState(Enum): # This recipe has not been cooked because it didn't need to be. SKIPPED = 10 + # This recipe failed. + FAILED = -1 + @staticmethod def is_completed(state): return state in (RecipeState.COOKED, RecipeState.SKIPPED) diff --git a/scone/head/dependency_tracking.py b/scone/head/dependency_tracking.py index 6fe8874..8da5ec0 100644 --- a/scone/head/dependency_tracking.py +++ b/scone/head/dependency_tracking.py @@ -1,6 +1,7 @@ import json import logging import time +from copy import deepcopy from hashlib import sha256 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union @@ -12,6 +13,7 @@ from aiosqlite import Connection from scone.head.dag import Resource from scone.head.recipe import recipe_name_getter +from scone.head.variables import Variables if TYPE_CHECKING: from scone.head.dag import RecipeDag @@ -118,7 +120,8 @@ class DependencyTracker: def register_variable(self, variable: str, value: Union[dict, str, int]): # self._vars[variable] = value - raise NotImplementedError("time") + # TODO(implement) + logger.critical("not implemented: register var %s", variable) def register_fridge_file(self, desugared_path: str): # TODO this is not complete @@ -131,19 +134,48 @@ class DependencyTracker: file_res = Resource("file", path, sous=sous) self.watch(file_res) - # def get_j2_compatible_dep_var_proxies( - # self, variables: Variables - # ) -> Dict[str, "DependencyVarProxy"]: - # # XXX BROKEN does not work for overrides - # result = {} - # - # if len("1"): - # raise NotImplementedError("BROKEN") - # - # for key, vars in variables.toplevel().items(): - # result[key] = DependencyVarProxy(self, vars, key + ".") - # - # return result + def get_j2_var_proxies( + self, variables: Variables + ) -> Dict[str, "DependencyVarProxy"]: + result = {} + + for key in variables.toplevel(): + result[key] = DependencyVarProxy(key, variables, self) + + return result + + +class DependencyVarProxy: + def __init__( + self, + current_path_prefix: Optional[str], + vars: Variables, + tracker: DependencyTracker, + ): + self._current_path_prefix = current_path_prefix + self._vars = vars + self._tracker = tracker + + def raw_(self) -> Dict[str, Any]: + if not self._current_path_prefix: + raw_dict = self._vars.toplevel() + else: + raw_dict = self._vars.get_dotted(self._current_path_prefix) + self._tracker.register_variable(self._current_path_prefix or "", raw_dict) + return deepcopy(raw_dict) + + def __getattr__(self, name: str) -> Union["DependencyVarProxy", Any]: + if not self._current_path_prefix: + dotted_path = name + else: + dotted_path = f"{self._current_path_prefix}.{name}" + raw_value = self._vars.get_dotted(dotted_path) + + if isinstance(raw_value, dict): + return DependencyVarProxy(dotted_path, self._vars, self._tracker) + else: + self._tracker.register_variable(dotted_path, raw_value) + return raw_value class DependencyCache: diff --git a/scone/head/dot_emitter.py b/scone/head/dot_emitter.py index 59d96ab..ee531bd 100644 --- a/scone/head/dot_emitter.py +++ b/scone/head/dot_emitter.py @@ -5,13 +5,14 @@ from scone.head.dag import RecipeDag, RecipeState, Resource, Vertex from scone.head.recipe import Recipe, recipe_name_getter state_to_colour = { - RecipeState.LOADED: "#000000", - RecipeState.PREPARED: "azure", - RecipeState.PENDING: "pink", - RecipeState.COOKABLE: "gold", - RecipeState.COOKED: "darkolivegreen1", - RecipeState.SKIPPED: "cadetblue1", - RecipeState.BEING_COOKED: "darkorange1", + RecipeState.LOADED: ("white", "black"), + RecipeState.PREPARED: ("azure", "black"), + RecipeState.PENDING: ("pink", "black"), + RecipeState.COOKABLE: ("gold", "black"), + RecipeState.COOKED: ("darkolivegreen1", "black"), + RecipeState.SKIPPED: ("cadetblue1", "black"), + RecipeState.BEING_COOKED: ("darkorange1", "black"), + RecipeState.FAILED: ("black", "orange"), } @@ -32,10 +33,10 @@ def emit_dot(dag: RecipeDag, path_out: Path) -> None: f"{recipe_name_getter(vertex.__class__)}" f" [{rec_meta.incoming_uncompleted}]" ) - colour = state_to_colour[rec_meta.state] + colour, text_colour = state_to_colour[rec_meta.state] fout.write( f'\t{vertex_id} [shape=box, label="{label}",' - f" style=filled, fillcolor={colour}];\n" + f" style=filled, fontcolor={text_colour}, fillcolor={colour}];\n" ) elif isinstance(vertex, Resource): label = str(vertex).replace("\\", "\\\\").replace('"', '\\"') diff --git a/scone/head/kitchen.py b/scone/head/kitchen.py index 3339930..8d98604 100644 --- a/scone/head/kitchen.py +++ b/scone/head/kitchen.py @@ -208,7 +208,11 @@ class Kitchen: self._dependency_trackers[next_job] = DependencyTracker( DependencyBook(), dag, next_job ) - await next_job.cook(self) + try: + await next_job.cook(self) + except Exception as e: + meta.state = RecipeState.FAILED + raise RuntimeError(f"Recipe {next_job} failed!") from e eprint(f"cooked {next_job}") # TODO cook # TODO store depbook diff --git a/scone/head/menu_reader.py b/scone/head/menu_reader.py index c038e5e..30854ca 100644 --- a/scone/head/menu_reader.py +++ b/scone/head/menu_reader.py @@ -10,11 +10,11 @@ import textx from scone.head.dag import RecipeDag, Resource from scone.head.recipe import RecipeContext +from scone.head.variables import Variables if typing.TYPE_CHECKING: from scone.head.head import Head from scone.head.recipe import Recipe - from scone.head.variables import Variables def _load_grammar(): @@ -69,10 +69,10 @@ class MenuBlock: user_directive: Optional[str] = None sous_directive: Optional[str] = None - for_directives: List[ForDirective] = [] - import_directives: List[str] = [] - recipe_edges: List[RecipeEdgeDirective] = [] - resource_edges: List[ResourceEdgeDirective] = [] + for_directives: List[ForDirective] = attr.ib(factory=list) + import_directives: List[str] = attr.ib(factory=list) + recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list) + resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list) @attr.s(auto_attribs=True, eq=False) @@ -89,9 +89,9 @@ class MenuRecipe: user_directive: Optional[str] = None sous_directive: Optional[str] = None - for_directives: List[ForDirective] = [] - recipe_edges: List[RecipeEdgeDirective] = [] - resource_edges: List[ResourceEdgeDirective] = [] + for_directives: List[ForDirective] = attr.ib(factory=list) + recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list) + resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list) def convert_textx_value(txvalue) -> Any: @@ -336,11 +336,13 @@ class MenuLoader: hierarchical_source=hierarchical_source, # XXX human=recipe.human, ) - args = recipe.arguments # noqa - # XXX sub in vars - instance: Recipe = recipe_class.new( - context, recipe.arguments, self._head - ) + try: + args = _vars.substitute_in_dict_copy(recipe.arguments) + except KeyError as ke: + raise KeyError( + f"When substituting for {hierarchical_source} / {recipe}" + ) from ke + instance: Recipe = recipe_class.new(context, args, self._head) self._recipes[recipe][(sous, for_indices)] = instance self._dag.add(instance) diff --git a/scone/head/variables.py b/scone/head/variables.py index 0bc779e..8c8d9be 100644 --- a/scone/head/variables.py +++ b/scone/head/variables.py @@ -1,5 +1,6 @@ +from copy import deepcopy from enum import Enum -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional, Set ExpressionPart = NamedTuple("ExpressionPart", [("kind", str), ("value", str)]) @@ -131,14 +132,14 @@ class Variables: self.set_dotted(var_name, sub_val) return sub_val else: - raise KeyError(f"No variable '{incoming}'") + raise KeyError(f"No variable '{var_name}'") out = "" for part in parsed: if part.kind == "literal": out += part.value elif part.kind == "variable": - var_name = parsed[0].value + var_name = part.value if self.has_dotted(var_name): out += str(self.get_dotted(var_name)) elif var_name in incoming: @@ -147,7 +148,7 @@ class Variables: self.set_dotted(var_name, sub_val) out += str(sub_val) else: - raise KeyError(f"No variable '{incoming}'") + raise KeyError(f"No variable '{var_name}'") return out def load_vars_with_substitutions(self, incoming: Dict[str, Any]): @@ -170,5 +171,16 @@ class Variables: elif isinstance(v, str): dictionary[k] = self.eval(v) + def substitute_in_dict_copy(self, dictionary: Dict[str, Any]): + new_dict = deepcopy(dictionary) + self.substitute_inplace_in_dict(new_dict) + return new_dict + def toplevel(self): return self._vars + + def keys(self) -> Set[str]: + keys = set(self._vars.keys()) + if self._delegate: + keys.update(self._delegate.keys()) + return keys