From b07a7c4bf08db2928e1845c4e60798f7c9ad8b65 Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Fri, 10 Sep 2010 20:24:47 +0200 Subject: [PATCH] Created documentation Unsing Sphinx, restructured text Updated documentation strings in config.py and config_parser.py Makefile for running nosetests and building docs --- t/Makefile | 7 ++ t/doc/Makefile | 89 ++++++++++++++++++ t/doc/conf.py | 198 +++++++++++++++++++++++++++++++++++++++++ t/doc/config.rst | 34 +++++++ t/doc/eval_timing.rst | 65 ++++++++++++++ t/doc/index.rst | 22 +++++ t/doc/log.rst | 11 +++ t/moe/config.py | 25 +++--- t/moe/config_parser.py | 55 ++++++------ 9 files changed, 464 insertions(+), 42 deletions(-) create mode 100644 t/Makefile create mode 100644 t/doc/Makefile create mode 100644 t/doc/conf.py create mode 100644 t/doc/config.rst create mode 100644 t/doc/eval_timing.rst create mode 100644 t/doc/index.rst create mode 100644 t/doc/log.rst diff --git a/t/Makefile b/t/Makefile new file mode 100644 index 0000000..4b7eeb7 --- /dev/null +++ b/t/Makefile @@ -0,0 +1,7 @@ +.PHONY: doc test + +doc: + cd doc; make html + +test: + nosetests moe.config_test diff --git a/t/doc/Makefile b/t/doc/Makefile new file mode 100644 index 0000000..a372cc7 --- /dev/null +++ b/t/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Moe.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Moe.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/t/doc/conf.py b/t/doc/conf.py new file mode 100644 index 0000000..30f2a77 --- /dev/null +++ b/t/doc/conf.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# +# Moe documentation build configuration file, created by +# sphinx-quickstart on Tue Aug 31 02:28:06 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +sys.path.append('..') + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage'] + +todo_include_todos = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Moe' +copyright = u'2010, MJ, gavento' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0' +# The full version, including alpha/beta/rc tags. +release = '0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'sphinxdoc' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = {'nosidebar':'true'} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Moedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Moe.tex', u'Moe Documentation', + u'MJ, gavento', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/t/doc/config.rst b/t/doc/config.rst new file mode 100644 index 0000000..57d58c1 --- /dev/null +++ b/t/doc/config.rst @@ -0,0 +1,34 @@ +Moe configuration +================= + +The configuration consists of lazy definition-style configuration resolver and configuration parser. + +.. contents:: + + +Config elements +---------------- + +.. automodule:: moe.config + :members: ConfigError, UndefinedError, VariableNameError, VariableFixedError, CyclicConfigError, ConfigTree, ConfigElem, ConfigCondition + :undoc-members: + + +Internal classes +++++++++++++++++ + +.. autoclass:: moe.config.Operation + :members: + :undoc-members: +.. autoclass:: moe.config.ConfigVar + :members: + :undoc-members: +.. autoclass:: moe.config.ConfigExpression + :members: + :undoc-members: + + +Config parser +-------------- + +.. automodule:: moe.config_parser diff --git a/t/doc/eval_timing.rst b/t/doc/eval_timing.rst new file mode 100644 index 0000000..c35ae12 --- /dev/null +++ b/t/doc/eval_timing.rst @@ -0,0 +1,65 @@ +Evaluation timeline +==================== + +Main pipeline +------------- + +======= ======================== +Time Action +======= ======================== +\ Apply command-line overrides, set basic builtins (`CONFIG_FILE`, `VERBOSITY`, ...) +\ FIX: `CONFIG_FILE` +\ Load `CONFIG_FILE` +\ FIX: `LOG_FILE`, `VERBOSITY`, `HOME`, `TDIR`, `BOXDIR`, `HOOKS`, `TEST_HOOKS`, ... +\ Start logging to `LOG_FILE` +\ Insert predefined hooks, `HOOKS` and `TEST_HOOKS` to pipelines +5 [m] Check basic settings, directories and rights +10 [m] Task **must** be ready (rsynced, pulled). Locate and fix `PDIR` +15 [m] Load `TASK_CONFIG_FILE` (dep. on `PDIR`) +20 [m] Check `TASK_TYPE` and depending on that, initialize the module and insert future actions +**For 'standard':** +-------------------------------- +25 [tt] Locate the source +30 [tt] Initialize sandbox module +35 [tt] Initialize compiler module, insert its hooks +50 [cpl] Compile the source, perhaps using the sandbox +60 [tt] Run test pipeline for each `TEST` of `TESTS` +**For 'opendata':** +-------------------------------- +25 [tt] Locate the data files (as a dir) +30 [tt] Initialize sandbox module (if needed) +40 [tt] Initialize judge module, insert its hooks +60 [tt] Run test pipeline for each `TEST` of `TESTS` +**Recovery** +-------------------------------- +70 Any failure jumps to this time +80 [*] Cleanup must be finished +\ Write metafile +======= ======================== + +Test pipeline +------------- + +======= ======================== +Time Action +======= ======================== +\ Tee logging to test log file +**For 'standard':** +-------------------------------- +10 [tt] Copy test data to `TDIR` and to `BOXDIR` +30 [tt] Run testee (inside sandbox) +40 [tt] Run judge (inside sandbox) +50 [tt] Read/parse judge result +60 [tt] Copy program output to `TDIR` +**For 'opendata':** +-------------------------------- +10 [tt] Copy test data to `TDIR` and to `BOXDIR` +20 [tt] Locate the data file, copy to +40 [tt] Run judge (inside sandbox) +50 [tt] Read/parse judge result +**Recovery** +-------------------------------- +70 Any failure jumps to this time +80 [*] Cleanup must be finished +\ Write metafile +======= ======================== diff --git a/t/doc/index.rst b/t/doc/index.rst new file mode 100644 index 0000000..d30fb17 --- /dev/null +++ b/t/doc/index.rst @@ -0,0 +1,22 @@ +================= +Moe documentation +================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + log.rst + config.rst + eval_timing.rst + +.. automodule:: moe.config + +.. Indices and tables + ================== + +.. * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` + diff --git a/t/doc/log.rst b/t/doc/log.rst new file mode 100644 index 0000000..c25ded4 --- /dev/null +++ b/t/doc/log.rst @@ -0,0 +1,11 @@ +Moe Logging +=========== + +One `Loggers` object is created for Eval, initializing 4 subloggers (see class description). + +Use as `e.log.debug(...)` for both test and main log, `e.log.user.debug(...)` for `user` and other logs. + +.. autoclass:: moe.log.Loggers + :members: + :member-order: bysource + diff --git a/t/moe/config.py b/t/moe/config.py index 813976c..f24e1ef 100644 --- a/t/moe/config.py +++ b/t/moe/config.py @@ -1,25 +1,22 @@ """ config.py -------- +--------- Lazy conditional string evaluation module for Moe configuration variables. * Each variable has ordered list of operations (definitions), each defining operation either -assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s). - -NOTE: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change. + assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s). * Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions. * Expression is a list of strings and variables to be expanded. -NOTE: All expanded data should be (or is converted to) unicode - - -TODO (OPT): Cleanup of unused undefined variables. -TODO (OPT): Better variable name checking (no name '.'-structural prefix of another) -TODO (OPT): Implemet "subtree" listing. +.. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change. +.. note:: All expanded data should be (or is converted to) unicode +.. todo:: (OPT) Cleanup of unused undefined variables. +.. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another) +.. todo:: (OPT) Implemet "subtree" listing. """ import types, itertools, re, bisect @@ -159,9 +156,11 @@ class ConfigElem(object): class ConfigCondition(ConfigElem): """ Condition using equality and logic operators. - Clause is a tuple-tree in the following recursive form: - ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), - ('==', e1, e2), ('!=', e1, e2) where e1, e2 are `ConfigExpression`s. + Clause is a tuple-tree in the following recursive form:: + + ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) + + where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`. """ def __init__(self, formula, text=None, parent=None): diff --git a/t/moe/config_parser.py b/t/moe/config_parser.py index 73b5406..217cd91 100644 --- a/t/moe/config_parser.py +++ b/t/moe/config_parser.py @@ -1,47 +1,44 @@ -""" +r""" config_parser.py ------------- +---------------- Simple Moe configuration file syntax parser. -TODO: decide neccessity of '()' in/around formulas -TODO: check escaping in expressions -TODO: should whitespace (incl. '\\n') be allowed (almost) everywhere? - can comment be anywhere whitespace can? - Generally, whitespace and comments are alowed everywhere except in variable names and inside expressions. -Also, COMMENT must not contain '\\n'. +Also, COMMENT must not contain '\n'. -FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace. TODO: check? +FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace. -The configuration syntax is the following: +The configuration syntax is the following:: -FILE = BLOCK -BLOCK = WS | STATEMENT ( SEP STATEMENT )* + FILE = BLOCK + BLOCK = WS | STATEMENT ( SEP STATEMENT )* -SEP = ( '\\n' | ';' ) -WS = ( ' ' | '\\t' | '\\n' | COMMENT )* + SEP = ( '\n' | ';' ) + WS = ( ' ' | '\t' | '\n' | COMMENT )* -COMMENT = re('#[^\\n]*\\n') + COMMENT = re('#[^\n]*\n') -STATEMENT = CONDITION | OPERATION | SUBTREE + STATEMENT = CONDITION | OPERATION | SUBTREE -OPERATION = WS VARNAME WS ( '=' | '+=' ) WS EXPRESSION -SUBTREE = WS VARNAME WS '{' BLOCK WS '}' -CONDITION = WS 'if' FORMULA WS '{' BLOCK WS '}' + OPERATION = WS VARNAME WS ( '=' | '+=' ) WS EXPRESSION + SUBTREE = WS VARNAME WS '{' BLOCK WS '}' + CONDITION = WS 'if' FORMULA WS '{' BLOCK WS '}' -FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | '(' AND WS ')' | '(' OR WS ')' | NOT ) -AND = FORMULA WS 'and' FORMULA -OR = FORMULA WS 'or' FORMULA -NOT = WS 'not' FORMULA + FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | '(' AND WS ')' | '(' OR WS ')' | NOT ) + AND = FORMULA WS 'and' FORMULA + OR = FORMULA WS 'or' FORMULA + NOT = WS 'not' FORMULA -NOTE: ';' or '\n' is currently required even after CONDITION and SUBTREE block - TODO: change to OPERATION only -NOTE: Formula may contain additional/extra parentheses + EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\n]*'" | VARNAME + ECHAR = re('([^\{}]|\\|\{|\}|\\n)*') + VARNAME = re('[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)*') -EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'" | VARNAME -ECHAR = re('([^\\{}]|\\\\|\\{|\\}|\\n)*') -VARNAME = re('[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)*') +.. todo:: should whitespace (incl. '\n') be allowed (almost) everywhere? + can comment be anywhere whitespace can? +.. note:: ';' or '\n' is currently required even after CONDITION and SUBTREE block +.. todo:: change to OPERATION only +.. note:: Formula can contain additional/unnecessary parentheses """ import re, types, itertools, logging as log -- 2.39.2