From f1197017785dc3b28835cad97de45db673304eb3 Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Tue, 23 Nov 2010 18:56:39 +0100 Subject: [PATCH] Rewrite docs of config Much better now (or even finished?!) --- t/doc/conf.py | 4 +- t/doc/config.rst | 209 ++++++++++++++++++++++++++++++++++------- t/moe/config.py | 39 ++++---- t/moe/config_parser.py | 13 ++- 4 files changed, 207 insertions(+), 58 deletions(-) diff --git a/t/doc/conf.py b/t/doc/conf.py index a23201d..eede646 100644 --- a/t/doc/conf.py +++ b/t/doc/conf.py @@ -24,7 +24,7 @@ sys.path.append('..') # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.inheritance_diagram'] todo_include_todos = True @@ -97,7 +97,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'sphinxdoc' +html_theme = 'agogo' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/t/doc/config.rst b/t/doc/config.rst index fe40662..3216a46 100644 --- a/t/doc/config.rst +++ b/t/doc/config.rst @@ -1,53 +1,198 @@ .. _config: +.. highlight:: none + +================= Moe configuration ================= - .. contents:: -The configuration consists of lazy definition-style configuration resolver and configuration parser. +------------------- +Configuration logic +------------------- -The configuration variables form a dot-separated tree, but internally the list of variables is just -flat with full names. +Moe and the derived modules use many configuration variables indicating i.e. working directories, +task name, compiler commandline and options, current test name and name of test file to use. +All values are treated as text and may contain unicode characters (although this is usually not advisable). +A variable may be either set to a simple fixed string or to a string containing expansions of other variables. +The configuration library also supports appending text, conditionals and a simple variable hierarchy. -Configuration tree -++++++++++++++++++ +For the complete configuration syntax grammar, see :mod:`moe.config_parser` documentation. + +Generally, both whitespace and indentation are optional and do not matter, individual +operations must be separated by either ``;`` or newline, ``#`` introduces a comment until the end of the line. + +In *moe*, the config files are assumed to be in UTF-8, but using non-ASCII characters are discouraged outside +comments (mainly because not all unix comands handle unicode well). + +Variables and substitution +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The names of the variables may consist of letters, numbers, underscore, dash and a dot. Dot acts as +a separator of the variable name hierarchy and may not occur at the beginning or the end of variable name, +nor adjacent to another dot. The tree hierarchy is provided only for user's convenience and does not play +any significant role in variable evaluation. + +The value of a variable may be defined by several operations: + +**SET**, ``VAR = "value"`` + Sets the variable to the variable-expansion of the string. + +**APPEND**, ``VAR += "value"`` + Appends the variable-expansion of the string. + +Value is a ``"``-delimited string that may contain substitutions in the form ``{VARNAME}`` i.e. +``"source-{VERSION}.{SUFFIX}"``, where ``VERSION`` and ``SUFFIX`` are the substituted variables. +The string may contain any characters including unicode (although this is not always advisable). +Characters ``"``, ``{``, ``{`` and ``\`` have to be escaped with ``\``, i.e. ``\"``, ``\{``, ``\{`` and ``\\``. + +When evaluating a variable, all the substitutions are evaluated as well. Every variable is evaluated +in the same way, independently of the priority of the operations it occurs in. The substitutions must +therefore be acyclic and a variable may not contain itself as a substitution. + + +Operation priority +^^^^^^^^^^^^^^^^^^ + +Each operation has a priority usually depending on its origin, i.e. builtins, global config file, task config file +or commandline overrides. +Only the **SET** operation with the highest priority and higher priority **APPENDs** are used (as if all operations were +applied from the least priority operation to the highest). +Operations of the same priority are processed by their definition order (first to last). + +For example, in builtins (priority 0) we may have:: + + COMPILE = "{CC} {OPTS} {INFILE} -o {OUTFILE}" + OPTS = "-O{OPTIMIZE}" + CC = "gcc" + +In config file (priority 30):: + + OPTS += " -W{WARNOPT}" # OPTS is now "-O{OPTIMIZE} -W{WARNOPT}" + WARNOPT = "all" + OPTIMIZE = "0" + +And in command-line overrides (priority 100):: + + COMPILE = "cp {INFILE} {OUTFILE}" + +Moe would then later set ``INFILE`` and ``OUTFILE`` at some point before evaluating and executing +the compilation command. + +With these *lazy* semantics, it is possible to set a variable to a value depending +on both higher- and lower-priority operations. + +.. note:: + The implementation is actually very efficient and recalculates only the necessary values, + caching the results. Adding/removing operations to variables only recursively invalidates the + (potentially) influenced cached values, these are recalculated lazily on-demand. + +Variables hierarchy +^^^^^^^^^^^^^^^^^^^ + +The variables form a tree-like hierarchy with levels separated by dots. The variables +may be defined either by providing the full name or by inside subtrees. +Subtrees provide all the *defined* variables with given dot-separated prefix. +Substituted variable names must be given by their full names. :: + + LOG.DIR = "./log" + LOG { + VERBOSE = "N" + # Defines LOG.VERBOSE + TEST.LOGFILE = "{LOG.DIR}/test.log" + # Defines LOG.TEST.LOGFILE + } + LOG.TEST { VERBOSE = "Y" } + # The subtree name may contain several level names + +Conditionals +^^^^^^^^^^^^ + +Any block of operations may be guarded by a condition. The conditions may be nested and +consist of expressions containing ``and``, ``or``, ``not`` and variable/string (in)equalities. The strings +may contain substitutions. + +The syntax is the following:: + + if (NAME == "{TASK}.cc") or (NAME == "{TASK}.cpp") or + ((LANG_HINT == "C") and (not FOO != "BAR")) + { + LANG = "C" + } + +The curly brackets delimiting the bloc, as well as the brackets in the boolena expression, +are mandatory. ``else`` is not supported. ``if VAR=="FOO" {...}`` is equivalent to ``if "{VAR}"=="FOO" {...}``. + + +------------------- +Module `moe.config` +------------------- +.. highlight:: python .. automodule:: moe.config - Exceptions - ---------- +Exceptions +^^^^^^^^^^ + +.. autoclass:: ConfigError +.. autoclass:: UndefinedError +.. autoclass:: VariableNameError +.. autoclass:: VariableFixedError +.. autoclass:: CyclicConfigError + +Configuration tree +^^^^^^^^^^^^^^^^^^ + +The configuration environment :class:`ConfigTree` is implemented as a flat dictionary of variables +referenced by their full name. + +.. autoclass:: ConfigTree + :members: + +Variables +^^^^^^^^^ + +Every variable object :class:`ConfigVar` stores the list of its defining operations, the current +cached value and both the variables its current value depends on and the variables that currently depend on it. +All the dependencies are calculated upon evaluation and removed upon invalidation. +Only the operations and conditions influencing the current value contribute to dependencies. + +.. note:: If no **SET** applies, a variable is still undefined even if some **APPEND** applies. This *might* change. + +.. autoclass:: ConfigVar + :members: + +Operations +^^^^^^^^^^ + +Every operation object is either **SET** or **APPEND** and may be guarded by a +condition. + +.. autoclass:: Operation + +Conditions +^^^^^^^^^^ + +Every condition is defined by a formula, the result is cached in the same way as with :class:`ConfigVar`, including +dependency tracking. A condition may depend on another parent condition in case this condition is +nested inside another. In this case the parent condition is checked first and the parent condition is a dependency. - .. autoclass:: ConfigError - .. autoclass:: UndefinedError - .. autoclass:: VariableNameError - .. autoclass:: VariableFixedError - .. autoclass:: CyclicConfigError +.. autoclass:: ConfigCondition + :members: - Configuration tree - ------------------ +Internals +^^^^^^^^^ - .. autoclass:: ConfigTree - :members: +.. autoclass:: ConfigElem + :members: - Main config elements - -------------------- +.. autoclass:: ConfigExpression - .. autoclass:: ConfigElem - :members: - .. autoclass:: ConfigCondition - :members: - .. autoclass:: ConfigVar - :members: - - Internal config elements - ------------------------ - .. autoclass:: Operation - .. autoclass:: ConfigExpression -Config parser -+++++++++++++ +---------------------------------------- +Config parser module `moe.config_parser` +---------------------------------------- .. automodule:: moe.config_parser diff --git a/t/moe/config.py b/t/moe/config.py index e6ce045..ec057e1 100644 --- a/t/moe/config.py +++ b/t/moe/config.py @@ -1,15 +1,6 @@ """ -Lazy conditional string evaluation module for Moe configuration variables. +Module for managing and evaluation of Moe configuration variables. -* Each variable has ordered list of operations (definitions), each defining operation either - assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s). - -* Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions. - -* Expression is a list of strings and variables to be expanded. - -.. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change. -.. note:: All expanded data should be (or is converted to) unicode .. todo:: (OPT) Cleanup of unused undefined variables. .. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another) .. todo:: (OPT) Implemet "subtree" listing. @@ -41,20 +32,26 @@ def check_depth(depth): class ConfigError(MoeError): + "Base class for moe.config errors" pass class UndefinedError(ConfigError): + "Raised when no **SET** operation applies to evaluated variable." pass class VariableNameError(ConfigError): + "Raised on invalid config variable name." pass class VariableFixedError(ConfigError): + "Raised when modifying a fixed variable" pass class CyclicConfigError(ConfigError): + "Raised when evaluation recursion is too deep" pass + class ParseProxy(list): """Proxy helper class around values returned by `parse` and `parse_file`, useful in "with" constructs.""" @@ -69,9 +66,7 @@ class ParseProxy(list): class ConfigTree(object): """ - Configuration tree containing all the variables. - - The variables in `self.variables` are referenced directly by the full name. + Configuration environment containing the variables. """ def __init__(self): @@ -155,7 +150,7 @@ class ConfigTree(object): class ConfigElem(object): """ - Base class for cahed config elements - variables and conditions + Base class for cached config elements - variables and conditions """ def __init__(self, name): @@ -196,11 +191,11 @@ class ConfigElem(object): class ConfigCondition(ConfigElem): """ Condition using equality and logic operators. - Clause is a tuple-tree in the following recursive form:: + Formula is a tuple-tree in the following recursive form:: ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) - where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`. + where ``e1``, ``e2`` are :class:`ConfigExpression`, ``c1``, ``c2``, :class:`ConfigCondition`. """ def __init__(self, formula, text=None, parent=None): @@ -278,10 +273,15 @@ class ConfigCondition(ConfigElem): class Operation(object): - "Helper class for operation data. Must not be present in more variables or present multiple times." + """ + Helper class for operation data. Must be present at most once in at most one variable. + + ``operation`` is either ``"SET"`` or ``"APPEND"``, ``condition`` is a :class:`ConfigCondition` instance or ``None``, + ``expression`` is a :class:`ConfigExpression` instance, ``level`` is the priority of the operation and ``source`` + is an informative string describing the operation origin. + """ def __init__(self, operation, condition, expression, level=0, source='?'): - # operation is currently 'SET' and 'APPEND' self.operation = operation self.condition = condition self.expression = expression @@ -294,6 +294,7 @@ class Operation(object): class ConfigVar(ConfigElem): + "Class representing a single configuration variable" def __init__(self, name): super(ConfigVar, self).__init__(name) @@ -304,7 +305,7 @@ class ConfigVar(ConfigElem): self.fixed_val = None def variables(self): - "Return a set of variables used in the expressions" + "Return a set of variables used in the expressions of the operations" if not self.operations: return set([]) return set.union(*[ op.expression.variables() for op in self.operations ]) diff --git a/t/moe/config_parser.py b/t/moe/config_parser.py index 6dec8c9..b3131cc 100644 --- a/t/moe/config_parser.py +++ b/t/moe/config_parser.py @@ -1,10 +1,13 @@ r""" Simple Moe configuration file syntax parser. -Generally, whitespace and comments are alowed everywhere except in variable names and inside expressions. -Also, COMMENT must not contain '\n'. +Generally, whitespace and comments are alowed everywhere except in variable names and inside expressions, +``\\n`` ends a ``COMMENT``. -FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace. +``FILE``, ``BLOCK``, ``STATEMENT``, ``OPERATION``, ``SUBTREE``, ``CONDITION``, ``FORMULA``, ``AND``, ``OR`` +and ``NOT`` ignore any preceding whitespace. + +.. highlight:: none The configuration syntax is the following:: @@ -22,7 +25,8 @@ The configuration syntax is the following:: SUBTREE = WS VARNAME WS '{' BLOCK WS '}' CONDITION = WS 'if' FORMULA WS '{' BLOCK WS '}' - FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | '(' AND WS ')' | '(' OR WS ')' | NOT ) + FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | + '(' AND WS ')' | '(' OR WS ')' | NOT ) AND = FORMULA WS 'and' FORMULA OR = FORMULA WS 'or' FORMULA NOT = WS 'not' FORMULA @@ -34,7 +38,6 @@ The configuration syntax is the following:: .. todo:: should whitespace (incl. '\n') be allowed (almost) everywhere? can comment be anywhere whitespace can? .. note:: ';' or '\n' is currently required even after CONDITION and SUBTREE block -.. todo:: change to OPERATION only .. note:: Formula can contain additional/unnecessary parentheses """ -- 2.39.5