From c6f6feed76e0101fed44c3b98c40170442de7996 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 25 Aug 2018 16:36:05 -0700 Subject: [PATCH] guide: Start of HCL usage guide This is guide-style documentation to introduce the different parts of HCL, as a complement to the reference documentation provided in godoc. --- guide/.gitignore | 2 + guide/Makefile | 20 +++++ guide/conf.py | 164 ++++++++++++++++++++++++++++++++++++ guide/go.rst | 27 ++++++ guide/go_decoding_gohcl.rst | 126 +++++++++++++++++++++++++++ guide/go_diagnostics.rst | 97 +++++++++++++++++++++ guide/go_parsing.rst | 64 ++++++++++++++ guide/index.rst | 34 ++++++++ guide/intro.rst | 108 ++++++++++++++++++++++++ guide/make.bat | 36 ++++++++ guide/requirements.txt | 3 + 11 files changed, 681 insertions(+) create mode 100644 guide/.gitignore create mode 100644 guide/Makefile create mode 100644 guide/conf.py create mode 100644 guide/go.rst create mode 100644 guide/go_decoding_gohcl.rst create mode 100644 guide/go_diagnostics.rst create mode 100644 guide/go_parsing.rst create mode 100644 guide/index.rst create mode 100644 guide/intro.rst create mode 100644 guide/make.bat create mode 100644 guide/requirements.txt diff --git a/guide/.gitignore b/guide/.gitignore new file mode 100644 index 0000000..ced5893 --- /dev/null +++ b/guide/.gitignore @@ -0,0 +1,2 @@ +env/* +_build/* diff --git a/guide/Makefile b/guide/Makefile new file mode 100644 index 0000000..01f3758 --- /dev/null +++ b/guide/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = HCL +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/guide/conf.py b/guide/conf.py new file mode 100644 index 0000000..168a469 --- /dev/null +++ b/guide/conf.py @@ -0,0 +1,164 @@ +import subprocess +import os.path + +# -- Project information ----------------------------------------------------- + +project = u'HCL' +copyright = u'2018, HashiCorp' +author = u'HashiCorp' + +git_version = subprocess.check_output(['git', 'describe', '--always']).strip() + +# The short X.Y version +version = unicode(git_version) +# The full version, including alpha/beta/rc tags +release = unicode(git_version) + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.todo', + 'sphinx.ext.githubpages', + 'sphinxcontrib.golangdomain', + 'sphinx.ext.autodoc', + 'autoapi.extension', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', 'env'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'HCLdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'HCL.tex', u'HCL Documentation', + u'HashiCorp', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'hcl', u'HCL Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'HCL', u'HCL Documentation', + author, 'HCL', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +autoapi_type = 'go' +autoapi_dirs = [ + os.path.join(os.path.dirname(__file__), '..', x) + for x in ['gohcl', 'hcl', 'hcldec', 'hcled', 'hclparse', 'hcltest', 'hclwrite'] +] +autoapi_root = 'api' +autoapi_add_toctree_entry = False +autoapi_keep_files = True +autoapi_generate_api_docs = False diff --git a/guide/go.rst b/guide/go.rst new file mode 100644 index 0000000..bb99d86 --- /dev/null +++ b/guide/go.rst @@ -0,0 +1,27 @@ +Using HCL in a Go application +============================= + +HCL is itself written in Go_ and currently it is primarily intended for use as +a library within other Go programs. + +This section describes a number of different ways HCL can be used to define +and process a configuration language within a Go program. For simple situations, +HCL can decode directly into Go ``struct`` values in a similar way as encoding +packages such as ``encoding/json`` and ``encoding/xml``. + +The HCL Go API also offers some alternative approaches however, for processing +languages that may be more complex or that include portions whose expected +structure cannot be determined until runtime. + +The following sections give an overview of different ways HCL can be used in +a Go program. + +.. toctree:: + :maxdepth: 1 + :caption: Sub-sections: + + go_parsing + go_diagnostics + go_decoding_gohcl + +.. _Go: https://golang.org/ diff --git a/guide/go_decoding_gohcl.rst b/guide/go_decoding_gohcl.rst new file mode 100644 index 0000000..44d4cae --- /dev/null +++ b/guide/go_decoding_gohcl.rst @@ -0,0 +1,126 @@ +.. go:package:: gohcl + +Decoding Into Native Go Values +============================== + +The most straightforward way to access the content of an HCL file is to +decode into native Go values using ``reflect``, similar to the technique used +by packages like ``encoding/json`` and ``encoding/xml``. + +Package ``gohcl`` provides functions for this sort of decoding. Function +``DecodeBody`` attempts to extract values from an HCL *body* and write them +into a Go value given as a pointer: + +.. code-block:: go + + type ServiceConfig struct { + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + ListenAddr string `hcl:"listen_addr"` + } + type Config struct { + IOMode string `hcl:"io_mode"` + Services []ServiceConfig `hcl:"service,block"` + } + + var c Config + moreDiags := gohcl.DecodeBody(f.Body, nil, &c) + diags = append(diags, moreDiags...) + +The above example decodes the *root body* of a file ``f``, presumably loaded +previously using a parser, into the variable ``c``. The field labels within +the struct types imply the schema of the expected language, which is a cut-down +version of the hypothetical language we showed in :ref:`intro`. + +The struct field labels consist of two comma-separated values. The first is +the name of the corresponding argument or block type as it will appear in +the input file, and the second is the type of element being named. If the +second value is omitted, it defaults to ``attr``, requesting an attribute. + +Nested blocks are represented by a struct or a slice of that struct, and the +special element type ``label`` within that struct declares that each instance +of that block type must be followed by one or more block labels. In the above +example, the ``service`` block type is defined to require two labels, named +``type`` and ``name``. For label fields in particular, the given name is used +only to refer to the particular label in error messages when the wrong number +of labels is used. + +By default, all declared attributes and blocks are considered to be required. +An optional value is indicated by making its field have a pointer type, in +which case ``nil`` is written to indicate the absense of the argument. + +The sections below discuss some additional decoding use-cases. For full details +on the `gohcl` package, see +`the godoc reference `_. + +Variables and Functions +----------------------- + +By default, arguments given in the configuration may use only literal values +and the built in expression language operators, such as arithmetic. + +The second argument to ``gohcl.DecodeBody``, shown as ``nil`` in the previous +example, allows the calling application to additionally offer variables and +functions for use in expressions. Its value is a pointer to an +``hcl.EvalContext``, which will be covered in more detail in the later section +:ref:`expression-eval`. For now, a simple example of making the id of the +current process available as a single variable called ``pid``: + +.. code-block:: go + + type Context struct { + Pid string + } + ctx := gohcl.EvalContext(&Context{ + Pid: os.Getpid() + }) + var c Config + moreDiags := gohcl.DecodeBody(f.Body, ctx, &c) + diags = append(diags, moreDiags...) + +``gohcl.EvalContext`` constructs an expression evaluation context from a Go +struct value, making the fields available as variables and the methods +available as functions, after transforming the field and method names such +that each word (starting with an uppercase letter) is all lowercase and +separated by underscores. + +.. code-block:: hcl + + name = "example-program (${pid})" + +Partial Decoding +---------------- + +In the examples so far, we've extracted the content from the entire input file +in a single call to ``DecodeBody``. This is sufficient for many simple +situations, but sometimes different parts of the file must be evaluated +separately. For example: + +* If different parts of the file must be evaluated with different variables + or functions available. + +* If the result of evaluating one part of the file is used to set variables + or functions in another part of the file. + +There are several ways to perform partial decoding with ``gohcl``, all of +which involve decoding into HCL's own types, such as ``hcl.Body``. + +The most general approach is to declare an additional struct field of type +``hcl.Body``, with the special field tag type ``remain``: + +.. code-block:: go + + type ServiceConfig struct { + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + ListenAddr string `hcl:"listen_addr"` + Remain hcl.Body `hcl:",remain"` + } + +When a ``remain`` field is present, any element of the input body that is +not matched is retained in a body saved into that field, which can then be +decoded in a later call, potentially with a different evaluation context. + +Another option is to decode an attribute into a value of type `hcl.Expression`, +which can then be evaluated separately as described in +:ref:`expression-eval`. diff --git a/guide/go_diagnostics.rst b/guide/go_diagnostics.rst new file mode 100644 index 0000000..a948542 --- /dev/null +++ b/guide/go_diagnostics.rst @@ -0,0 +1,97 @@ +.. _go-diagnostics: + +Diagnostic Messages +=================== + +An important concern for any machine language intended for human authoring is +to produce good error messages when the input is somehow invalid, or has +other problems. + +HCL uses *diagnostics* to describe problems in an end-user-oriented manner, +such that the calling application can render helpful error or warning messages. +The word "diagnostic" is a general term that covers both errors and warnings, +where errors are problems that prevent complete processing while warnings are +possible concerns that do not block processing. + +HCL deviates from usual Go API practice by returning its own ``hcl.Diagnostics`` +type, instead of Go's own ``error`` type. This allows functions to return +warnings without accompanying errors while not violating the usual expectation +that the absense of errors is indicated by a nil ``error``. + +In order to easily accumulate and return multiple diagnostics at once, the +usual pattern for functions returning diagnostics is to gather them in a +local variable and then return it at the end of the function, or possibly +earlier if the function cannot continue due to the problems. + +.. code-block:: go + + func returningDiagnosticsExample() hcl.Diagnostics { + var diags hcl.Diagnostics + + // ... + + // Call a function that may itself produce diagnostics. + f, moreDiags := parser.LoadHCLFile("example.conf") + // always append, in case warnings are present + diags = append(diags, moreDiags...) + if diags.HasErrors() { + // If we can't safely continue in the presence of errors here, we + // can optionally return early. + return diags + } + + // ... + + return diags + } + +A common variant of the above pattern is calling another diagnostics-generating +function in a loop, using ``continue`` to begin the next iteration when errors +are detected, but still completing all iterations and returning the union of +all of the problems encountered along the way. + +In :ref:`go-parsing`, we saw that the parser can generate diagnostics which +are related to syntax problems within the loaded file. Further steps to decode +content from the loaded file can also generate diagnostics related to *semantic* +problems within the file, such as invalid expressions or type mismatches, and +so a program using HCL will generally need to accumulate diagnostics across +these various steps and then render them in the application UI somehow. + +Rendering Diagnostics in the UI +------------------------------- + +The best way to render diagnostics to an end-user will depend a lot on the +type of application: they might be printed into a terminal, written into a +log for later review, or even shown in a GUI. + +HCL leaves the responsibility for rendering diagnostics to the calling +application, but since rendering to a terminal is a common case for command-line +tools, the `hcl` package contains a default implementation of this in the +form of a "diagnostic text writer": + +.. code-block:: go + + wr := hcl.NewDiagnosticTextWriter( + os.Stdout, // writer to send messages to + parser.Files(), // the parser's file cache, for source snippets + 78, // wrapping width + true, // generate colored/highlighted output + ) + wr.WriteDiagnostics(diags) + +This default implementation of diagnostic rendering includes relevant lines +of source code for context, like this: + +:: + + Error: Unsupported block type + + on example.tf line 4, in resource "aws_instance" "example": + 2: provisionr "local-exec" { + + Blocks of type "provisionr" are not expected here. Did you mean "provisioner"? + +If the "color" flag is enabled, the severity will be additionally indicated by +a text color and the relevant portion of the source code snippet will be +underlined to draw further attention. + diff --git a/guide/go_parsing.rst b/guide/go_parsing.rst new file mode 100644 index 0000000..064bdee --- /dev/null +++ b/guide/go_parsing.rst @@ -0,0 +1,64 @@ +.. _go-parsing: + +Parsing HCL Input +================= + +The first step in processing HCL input provided by a user is to parse it. +Parsing turns the raw bytes from an input file into a higher-level +representation of the arguments and blocks, ready to be *decoded* into an +application-specific form. + +The main entry point into HCL parsing is :go:pkg:`hclparse`, which provides +:go:type:`hclparse.Parser`: + +.. code-block:: go + + parser := hclparse.NewParser() + f, diags := parser.ParseHCLFile("server.conf") + +Variable ``f`` is then a pointer to an :go:type:`hcl.File`, which is an +opaque abstract representation of the file, ready to be decoded. + +Variable ``diags`` describes any errors or warnings that were encountered +during processing; HCL conventionally uses this in place of the usual ``error`` +return value in Go, to allow returning a mixture of multiple errors and +warnings together with enough information to present good error messages to the +user. We'll cover this in more detail in the next section, +:ref:`go-diagnostics`. + +.. go:package:: hclparse + +Package ``hclparse`` +-------------------- + +.. go:type:: Parser + + .. go:function:: func NewParser() *Parser + + Constructs a new parser object. Each parser contains a cache of files + that have already been read, so repeated calls to load the same file + will return the same object. + + .. go:function:: func (*Parser) ParseHCL(src []byte, filename string) (*hcl.File, hcl.Diagnostics) + + Parse the given source code as HCL native syntax, saving the result into + the parser's file cache under the given filename. + + .. go:function:: func (*Parser) ParseHCLFile(filename string) (*hcl.File, hcl.Diagnostics) + + Parse the contents of the given file as HCL native syntax. This is a + convenience wrapper around ParseHCL that first reads the file into memory. + + .. go:function:: func (*Parser) ParseJSON(src []byte, filename string) (*hcl.File, hcl.Diagnostics) + + Parse the given source code as JSON syntax, saving the result into + the parser's file cache under the given filename. + + .. go:function:: func (*Parser) ParseJSONFile(filename string) (*hcl.File, hcl.Diagnostics) + + Parse the contents of the given file as JSON syntax. This is a + convenience wrapper around ParseJSON that first reads the file into memory. + +The above list just highlights the main functions in this package. +For full documentation, see +`the hclparse godoc `_. diff --git a/guide/index.rst b/guide/index.rst new file mode 100644 index 0000000..763e011 --- /dev/null +++ b/guide/index.rst @@ -0,0 +1,34 @@ +HCL Configuration Language +========================== + +HCL is a toolkit for creating structured configuration languages that are both +human- and machine-friendly, for use with command-line tools, servers, etc. + +HCL has both a native syntax, intended to be pleasant to read and write for +humans, and a JSON-based variant that is easier for machines to generate and +parse. The native syntax is inspired by libucl_, `nginx configuration`_, and +others. + +It includes an expression syntax that allows basic inline computation and, with +support from the calling application, use of variables and functions for more +dynamic configuration languages. + +HCL provides a set of constructs that can be used by a calling application to +construct a configuration language. The application defines which argument +names and nested block types are expected, and HCL parses the configuration +file, verifies that it conforms to the expected structure, and returns +high-level objects that the application can use for further processing. + +At present, HCL is primarily intended for use in applications written in Go_, +via its library API. + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + intro + go + +.. _libucl: https://github.com/vstakhov/libucl +.. _`nginx configuration`: http://nginx.org/en/docs/beginners_guide.html#conf_structure +.. _Go: https://golang.org/ diff --git a/guide/intro.rst b/guide/intro.rst new file mode 100644 index 0000000..d089a11 --- /dev/null +++ b/guide/intro.rst @@ -0,0 +1,108 @@ +.. _intro: + +Introduction to HCL +=================== + +HCL-based configuration is built from two main constructs: arguments and +blocks. The following is an example of a configuration language for a +hypothetical application: + +.. code-block:: hcl + + io_mode = "async" + + service "http" "web_proxy" { + listen_addr = "127.0.0.1:8080" + + process "main" { + command = ["/usr/local/bin/awesome-app", "server"] + } + + process "mgmt" { + command = ["/usr/local/bin/awesome-app", "mgmt"] + } + } + +In the above example, ``io_mode`` is a top-level argument, while ``service`` +introduces a block. Within the body of a block, further arguments and nested +blocks are allowed. A block type may also expect a number of *labels*, which +are the quoted names following the ``service`` keyword in the above example. + +The specific keywords ``io_mode``, ``service``, ``process``, etc here are +application-defined. HCL provides the general block structure syntax, and +can validate and decode configuration based on the application's provided +schema. + +HCL is a structured configuration language rather than a data structure +serialization language. This means that unlike languages such as JSON, YAML, +or TOML, HCL is always decoded using an application-defined schema. + +However, HCL does have a JSON-based alternative syntax, which allows the same +structure above to be generated using a standard JSON serializer when users +wish to generate configuration programmatically rather than hand-write it: + +.. code-block:: json + + { + "io_mode": "async", + "service": { + "http": { + "web_proxy": { + "listen_addr": "127.0.0.1:8080", + "process": { + "main": { + "command": ["/usr/local/bin/awesome-app", "server"] + }, + "mgmt": { + "command": ["/usr/local/bin/awesome-app", "mgmt"] + }, + } + } + } + } + } + +The calling application can choose which syntaxes to support. JSON syntax may +not be important or desirable for certain applications, but it is available for +applications that need it. The schema provided by the calling application +allows JSON input to be properly decoded even though JSON syntax is ambiguous +in various ways, such as whether a JSON object is representing a nested block +or an object expression. + +The collection of arguments and blocks at a particular nesting level is called +a *body*. A file always has a root body containing the top-level elements, +and each block also has its own body representing the elements within it. + +The term "attribute" can also be used to refer to what we've called an +"argument" so far. The term "attribute" is also used for the fields of an +object value in argument expressions, and so "argument" is used to refer +specifically to the type of attribute that appears directly within a body. + +The above examples show the general "texture" of HCL-based configuration. The +full details of the syntax are covered in the language specifications. + +.. todo:: Once the language specification documents have settled into a + final location, link them from above. + +Argument Expressions +-------------------- + +The value of an argument can be a literal value shown above, or it may be an +expression to allow arithmetic, deriving one value from another, etc. + +.. code-block:: hcl + + listen_addr = env.LISTEN_ADDR + +Built-in arithmetic and comparison operators are automatically available in all +HCL-based configuration languages. A calling application may optionally +provide variables that users can reference, like ``env`` in the above example, +and custom functions to transform values in application-specific ways. + +Full details of the expression syntax are in the HCL native syntax +specification. Since JSON does not have an expression syntax, JSON-based +configuration files use the native syntax expression language embedded inside +JSON strings. + +.. todo:: Once the language specification documents have settled into a + final location, link to the native syntax specification from above. diff --git a/guide/make.bat b/guide/make.bat new file mode 100644 index 0000000..08ad4e0 --- /dev/null +++ b/guide/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=HCL + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/guide/requirements.txt b/guide/requirements.txt new file mode 100644 index 0000000..421475a --- /dev/null +++ b/guide/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinxcontrib-golangdomain +sphinx-autoapi