diff --git a/guide/.gitignore b/guide/.gitignore new file mode 100644 index 0000000..ced5893 --- /dev/null +++ b/guide/.gitignore @@ -0,0 +1,2 @@ +env/* +_build/* diff --git a/guide/Makefile b/guide/Makefile new file mode 100644 index 0000000..01f3758 --- /dev/null +++ b/guide/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = HCL +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/guide/conf.py b/guide/conf.py new file mode 100644 index 0000000..099cdcd --- /dev/null +++ b/guide/conf.py @@ -0,0 +1,157 @@ +import subprocess +import os +import os.path + +# -- Project information ----------------------------------------------------- + +project = u'HCL' +copyright = u'2018, HashiCorp' +author = u'HashiCorp' + +if 'READTHEDOCS_VERSION' in os.environ: + version_str = os.environ['READTHEDOCS_VERSION'] +else: + version_str = subprocess.check_output(['git', 'describe', '--always']).strip() + +# The short X.Y version +version = unicode(version_str) +# The full version, including alpha/beta/rc tags +release = unicode(version_str) + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.todo', + 'sphinx.ext.githubpages', + 'sphinxcontrib.golangdomain', + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', 'env'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'HCLdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'HCL.tex', u'HCL Documentation', + u'HashiCorp', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'hcl', u'HCL Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'HCL', u'HCL Documentation', + author, 'HCL', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/guide/go.rst b/guide/go.rst new file mode 100644 index 0000000..bd6cef1 --- /dev/null +++ b/guide/go.rst @@ -0,0 +1,31 @@ +Using HCL in a Go application +============================= + +HCL is itself written in Go_ and currently it is primarily intended for use as +a library within other Go programs. + +This section describes a number of different ways HCL can be used to define +and process a configuration language within a Go program. For simple situations, +HCL can decode directly into Go ``struct`` values in a similar way as encoding +packages such as ``encoding/json`` and ``encoding/xml``. + +The HCL Go API also offers some alternative approaches however, for processing +languages that may be more complex or that include portions whose expected +structure cannot be determined until runtime. + +The following sections give an overview of different ways HCL can be used in +a Go program. + +.. toctree:: + :maxdepth: 1 + :caption: Sub-sections: + + go_parsing + go_diagnostics + go_decoding_gohcl + go_decoding_hcldec + go_expression_eval + go_decoding_lowlevel + go_patterns + +.. _Go: https://golang.org/ diff --git a/guide/go_decoding_gohcl.rst b/guide/go_decoding_gohcl.rst new file mode 100644 index 0000000..1a0d6e7 --- /dev/null +++ b/guide/go_decoding_gohcl.rst @@ -0,0 +1,130 @@ +.. go:package:: gohcl + +.. _go-decoding-gohcl: + +Decoding Into Native Go Values +============================== + +The most straightforward way to access the content of an HCL file is to +decode into native Go values using ``reflect``, similar to the technique used +by packages like ``encoding/json`` and ``encoding/xml``. + +Package ``gohcl`` provides functions for this sort of decoding. Function +``DecodeBody`` attempts to extract values from an HCL *body* and write them +into a Go value given as a pointer: + +.. code-block:: go + + type ServiceConfig struct { + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + ListenAddr string `hcl:"listen_addr"` + } + type Config struct { + IOMode string `hcl:"io_mode"` + Services []ServiceConfig `hcl:"service,block"` + } + + var c Config + moreDiags := gohcl.DecodeBody(f.Body, nil, &c) + diags = append(diags, moreDiags...) + +The above example decodes the *root body* of a file ``f``, presumably loaded +previously using a parser, into the variable ``c``. The field labels within +the struct types imply the schema of the expected language, which is a cut-down +version of the hypothetical language we showed in :ref:`intro`. + +The struct field labels consist of two comma-separated values. The first is +the name of the corresponding argument or block type as it will appear in +the input file, and the second is the type of element being named. If the +second value is omitted, it defaults to ``attr``, requesting an attribute. + +Nested blocks are represented by a struct or a slice of that struct, and the +special element type ``label`` within that struct declares that each instance +of that block type must be followed by one or more block labels. In the above +example, the ``service`` block type is defined to require two labels, named +``type`` and ``name``. For label fields in particular, the given name is used +only to refer to the particular label in error messages when the wrong number +of labels is used. + +By default, all declared attributes and blocks are considered to be required. +An optional value is indicated by making its field have a pointer type, in +which case ``nil`` is written to indicate the absense of the argument. + +The sections below discuss some additional decoding use-cases. For full details +on the `gohcl` package, see +`the godoc reference `_. + +.. _go-decoding-gohcl-evalcontext: + +Variables and Functions +----------------------- + +By default, arguments given in the configuration may use only literal values +and the built in expression language operators, such as arithmetic. + +The second argument to ``gohcl.DecodeBody``, shown as ``nil`` in the previous +example, allows the calling application to additionally offer variables and +functions for use in expressions. Its value is a pointer to an +``hcl.EvalContext``, which will be covered in more detail in the later section +:ref:`go-expression-eval`. For now, a simple example of making the id of the +current process available as a single variable called ``pid``: + +.. code-block:: go + + type Context struct { + Pid string + } + ctx := gohcl.EvalContext(&Context{ + Pid: os.Getpid() + }) + var c Config + moreDiags := gohcl.DecodeBody(f.Body, ctx, &c) + diags = append(diags, moreDiags...) + +``gohcl.EvalContext`` constructs an expression evaluation context from a Go +struct value, making the fields available as variables and the methods +available as functions, after transforming the field and method names such +that each word (starting with an uppercase letter) is all lowercase and +separated by underscores. + +.. code-block:: hcl + + name = "example-program (${pid})" + +Partial Decoding +---------------- + +In the examples so far, we've extracted the content from the entire input file +in a single call to ``DecodeBody``. This is sufficient for many simple +situations, but sometimes different parts of the file must be evaluated +separately. For example: + +* If different parts of the file must be evaluated with different variables + or functions available. + +* If the result of evaluating one part of the file is used to set variables + or functions in another part of the file. + +There are several ways to perform partial decoding with ``gohcl``, all of +which involve decoding into HCL's own types, such as ``hcl.Body``. + +The most general approach is to declare an additional struct field of type +``hcl.Body``, with the special field tag type ``remain``: + +.. code-block:: go + + type ServiceConfig struct { + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + ListenAddr string `hcl:"listen_addr"` + Remain hcl.Body `hcl:",remain"` + } + +When a ``remain`` field is present, any element of the input body that is +not matched is retained in a body saved into that field, which can then be +decoded in a later call, potentially with a different evaluation context. + +Another option is to decode an attribute into a value of type `hcl.Expression`, +which can then be evaluated separately as described in +:ref:`expression-eval`. diff --git a/guide/go_decoding_hcldec.rst b/guide/go_decoding_hcldec.rst new file mode 100644 index 0000000..db0e874 --- /dev/null +++ b/guide/go_decoding_hcldec.rst @@ -0,0 +1,242 @@ +.. go:package:: hcldec + +.. _go-decoding-hcldec: + +Decoding With Dynamic Schema +============================ + +In section :ref:`go-decoding-gohcl`, we saw the most straightforward way to +access the content from an HCL file, decoding directly into a Go value whose +type is known at application compile time. + +For some applications, it is not possible to know the schema of the entire +configuration when the application is built. For example, `HashiCorp Terraform`_ +uses HCL as the foundation of its configuration language, but parts of the +configuration are handled by plugins loaded dynamically at runtime, and so +the schemas for these portions cannot be encoded directly in the Terraform +source code. + +HCL's ``hcldec`` package offers a different approach to decoding that allows +schemas to be created at runtime, and the result to be decoded into +dynamically-typed data structures. + +The sections below are an overview of the main parts of package ``hcldec``. +For full details, see +`the package godoc `_. + +.. _`HashiCorp Terraform`: https://www.terraform.io/ + +Decoder Specification +--------------------- + +Whereas :go:pkg:`gohcl` infers the expected schema by using reflection against +the given value, ``hcldec`` obtains schema through a decoding *specification*, +which is a set of instructions for mapping HCL constructs onto a dynamic +data structure. + +The ``hcldec`` package contains a number of different specifications, each +implementing :go:type:`hcldec.Spec` and having a ``Spec`` suffix on its name. +Each spec has two distinct functions: + +* Adding zero or more validation constraints on the input configuration file. + +* Producing a result value based on some elements from the input file. + +The most common pattern is for the top-level spec to be a +:go:type:`hcldec.ObjectSpec` with nested specifications defining either blocks +or attributes, depending on whether the configuration file will be +block-structured or flat. + +.. code-block:: go + + spec := hcldec.ObjectSpec{ + "io_mode": &hcldec.AttrSpec{ + Name: "io_mode", + Type: cty.String, + }, + "services": &hcldec.BlockMapSpec{ + TypeName: "service", + LabelNames: []string{"type", "name"}, + Nested: hcldec.ObjectSpec{ + "listen_addr": &hcldec.AttrSpec{ + Name: "listen_addr", + Type: cty.String, + Required: true, + }, + "processes": &hcldec.BlockMapSpec{ + TypeName: "service", + LabelNames: []string{"name"}, + Nested: hcldec.ObjectSpec{ + "command": &hcldec.AttrSpec{ + Name: "command", + Type: cty.List(cty.String), + Required: true, + }, + }, + }, + }, + }, + } + val, moreDiags := hcldec.Decode(f.Body, spec, nil) + diags = append(diags, moreDiags...) + +The above specification expects a configuration shaped like our example in +:ref:`intro`, and calls for it to be decoded into a dynamic data structure +that would have the following shape if serialized to JSON: + +.. code-block:: JSON + + { + "io_mode": "async", + "services": { + "http": { + "web_proxy": { + "listen_addr": "127.0.0.1:8080", + "processes": { + "main": { + "command": ["/usr/local/bin/awesome-app", "server"] + }, + "mgmt": { + "command": ["/usr/local/bin/awesome-app", "mgmt"] + } + } + } + } + } + } + +.. go:package:: cty + +Types and Values With ``cty`` +----------------------------- + +HCL's expression interpreter is implemented in terms of another library called +:go:pkg:`cty`, which provides a type system which HCL builds on and a robust +representation of dynamic values in that type system. You could think of +:go:pkg:`cty` as being a bit like Go's own :go:pkg:`reflect`, but for the +results of HCL expressions rather than Go programs. + +The full details of this system can be found in +`its own repository `_, but this section +will cover the most important highlights, because ``hcldec`` specifications +include :go:pkg:`cty` types (as seen in the above example) and its results are +:go:pkg:`cty` values. + +``hcldec`` works directly with :go:pkg:`cty` — as opposed to converting values +directly into Go native types — because the functionality of the :go:pkg:`cty` +packages then allows further processing of those values without any loss of +fidelity or range. For example, :go:pkg:`cty` defines a JSON encoding of its +values that can be decoded losslessly as long as both sides agree on the value +type that is expected, which is a useful capability in systems where some sort +of RPC barrier separates the main program from its plugins. + +Types are instances of :go:type:`cty.Type`, and are constructed from functions +and variables in :go:pkg:`cty` as shown in the above example, where the string +attributes are typed as ``cty.String``, which is a primitive type, and the list +attribute is typed as ``cty.List(cty.String)``, which constructs a new list +type with string elements. + +Values are instances of :go:type:`cty.Value`, and can also be constructed from +functions in :go:pkg:`cty`, using the functions that include ``Val`` in their +names or using the operation methods available on :go:type:`cty.Value`. + +In most cases you will eventually want to use the resulting data as native Go +types, to pass it to non-:go:pkg:`cty`-aware code. To do this, see the guides +on +`Converting between types `_ +(staying within :go:pkg:`cty`) and +`Converting to and from native Go values `_. + +Partial Decoding +---------------- + +Because the ``hcldec`` result is always a value, the input is always entirely +processed in a single call, unlike with :go:pkg:`gohcl`. + +However, both :go:pkg:`gohcl` and :go:pkg:`hcldec` take :go:type:`hcl.Body` as +the representation of input, and so it is possible and common to mix them both +in the same program. + +A common situation is that :go:pkg:`gohcl` is used in the main program to +decode the top level of configuration, which then allows the main program to +determine which plugins need to be loaded to process the leaf portions of +configuration. In this case, the portions that will be interpreted by plugins +are retained as opaque :go:type:`hcl.Body` until the plugins have been loaded, +and then each plugin provides its :go:type:`hcldec.Spec` to allow decoding the +plugin-specific configuration into a :go:type:`cty.Value` which be +transmitted to the plugin for further processing. + +In our example from :ref:`intro`, perhaps each of the different service types +is managed by a plugin, and so the main program would decode the block headers +to learn which plugins are needed, but process the block bodies dynamically: + +.. code-block:: go + + type ServiceConfig struct { + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + PluginConfig hcl.Body `hcl:",remain"` + } + type Config struct { + IOMode string `hcl:"io_mode"` + Services []ServiceConfig `hcl:"service,block"` + } + + var c Config + moreDiags := gohcl.DecodeBody(f.Body, nil, &c) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + // (show diags in the UI) + return + } + + for _, sc := range c.Services { + pluginName := block.Type + + // Totally-hypothetical plugin manager (not part of HCL) + plugin, err := pluginMgr.GetPlugin(pluginName) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ /* ... */ }) + continue + } + spec := plugin.ConfigSpec() // returns hcldec.Spec + + // Decode the block body using the plugin's given specification + configVal, moreDiags := hcldec.Decode(sc.PluginConfig, spec, nil) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + + // Again, hypothetical API within your application itself, and not + // part of HCL. Perhaps plugin system serializes configVal as JSON + // and sends it over to the plugin. + svc := plugin.NewService(configVal) + serviceMgr.AddService(sc.Name, svc) + } + + +Variables and Functions +----------------------- + +The final argument to ``hcldec.Decode`` is an expression evaluation context, +just as with ``gohcl.DecodeBlock``. + +This object can be constructed using +:ref:`the gohcl helper function ` as before if desired, but +you can also choose to work directly with :go:type:`hcl.EvalContext` as +discussed in :ref:`go-expression-eval`: + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "pid": cty.NumberIntVal(int64(os.Getpid())), + }, + } + val, moreDiags := hcldec.Decode(f.Body, spec, ctx) + diags = append(diags, moreDiags...) + +As you can see, this lower-level API also uses :go:pkg:`cty`, so it can be +particularly convenient in situations where the result of dynamically decoding +one block must be available to expressions in another block. diff --git a/guide/go_decoding_lowlevel.rst b/guide/go_decoding_lowlevel.rst new file mode 100644 index 0000000..0f9662a --- /dev/null +++ b/guide/go_decoding_lowlevel.rst @@ -0,0 +1,199 @@ +.. _go-decoding-lowlevel: + +Advanced Decoding With The Low-level API +======================================== + +In previous sections we've discussed :go:pkg:`gohcl` and :go:pkg:`hcldec`, +which both deal with decoding of HCL bodies and the expressions within them +using a high-level description of the expected configuration schema. +Both of these packages are implemented in terms of HCL's low-level decoding +interfaces, which we will explore in this section. + +HCL decoding in the low-level API has two distinct phases: + +* Structural decoding: analyzing the arguments and nested blocks present in a + particular body. + +* Expression evaluation: obtaining final values for each argument expression + found during structural decoding. + +The low-level API gives the calling application full control over when each +body is decoded and when each expression is evaluated, allowing for more +complex configuration formats where e.g. different variables are available in +different contexts, or perhaps expressions within one block can refer to +values defined in another block. + +The low-level API also gives more detailed access to source location +information for decoded elements, and so may be desirable for applications that +do a lot of additional validation of decoded data where more specific source +locations lead to better diagnostic messages. + +Since all of the decoding mechanisms work with the same :go:type:`hcl.Body` +type, it is fine and expected to mix them within an application to get access +to the more detailed information where needed while using the higher-level APIs +for the more straightforward portions of a configuration language. + +The following subsections will give an overview of the low-level API. For full +details, see `the godoc reference `_. + +Structural Decoding +------------------- + +As seen in prior sections, :go:type:`hcl.Body` is an opaque representation of +the arguments and child blocks at a particular nesting level. An HCL file has +a root body containing the top-level elements, and then each nested block has +its own body presenting its own content. + +:go:type:`hcl.Body` is a Go interface whose methods serve as the structural +decoding API: + +.. go:currentpackage:: hcl + +.. go:type:: Body + + Represents the structural elements at a particular nesting level. + + .. go:function:: func (b Body) Content(schema *BodySchema) (*BodyContent, Diagnostics) + + Decode the content from the receiving body using the given schema. The + schema is considered exhaustive of all content within the body, and so + any elements not covered by the schema will generate error diagnostics. + + .. go:function:: func (b Body) PartialContent(schema *BodySchema) (*BodyContent, Body, Diagnostics) + + Similar to `Content`, but allows for additional arguments and block types + that are not described in the given schema. The additional body return + value is a special body that contains only the *remaining* elements, after + extraction of the ones covered by the schema. This returned body can be + used to decode the remaining content elsewhere in the calling program. + + .. go:function:: func (b Body) JustAttributes() (Attributes, Diagnostics) + + Decode the content from the receving body in a special *attributes-only* + mode, allowing the calling application to enumerate the arguments given + inside the body without needing to predict them in schema. + + When this method is used, a body can be treated somewhat like a map + expression, but it still has a rigid structure where the arguments must + be given directly with no expression evaluation. This is an advantage for + declarations that must themselves be resolved before expression + evaluation is possible. + + If the body contains any blocks, error diagnostics are returned. JSON + syntax relies on schema to distinguish arguments from nested blocks, and + so a JSON body in attributes-only mode will treat all JSON object + properties as arguments. + + .. go:function:: func (b Body) MissingItemRange() Range + + Returns a source range that points to where an absent required item in + the body might be placed. This is a "best effort" sort of thing, required + only to be somewhere inside the receving body, as a way to give source + location information for a "missing required argument" sort of error. + +The main content-decoding methods each require a :go:type:`hcl.BodySchema` +object describing the expected content. The fields of this type describe the +expected arguments and nested block types respectively: + +.. code-block:: go + + schema := &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "io_mode", + Required: false, + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "service", + LabelNames: []string{"type", "name"}, + }, + }, + } + content, moreDiags := body.Content(schema) + diags = append(diags, moreDiags...) + +:go:type:`hcl.BodyContent` is the result of both ``Content`` and +``PartialContent``, giving the actual attributes and nested blocks that were +found. Since arguments are uniquely named within a body and unordered, they +are returned as a map. Nested blocks are ordered and may have many instances +of a given type, so they are returned all together in a single slice for +further interpretation by the caller. + +Unlike the two higher-level approaches, the low-level API *always* works only +with one nesting level at a time. Decoding a nested block returns the "header" +for that block, giving its type and label values, but its body remains an +:go:type:`hcl.Body` for later decoding. + +Each returned attribute corresponds to one of the arguments in the body, and +it has an :go:type:`hcl.Expression` object that can be used to obtain a value +for the argument during expression evaluation, as described in the next +section. + +Expression Evaluation +--------------------- + +Expression evaluation *in general* has its own section, imaginitively titled +:ref:`go-expression-eval`, so this section will focus only on how it is +achieved in the low-level API. + +All expression evaluation in the low-level API starts with an +:go:type:`hcl.Expression` object. This is another interface type, with various +implementations depending on the expression type and the syntax it was parsed +from. + +.. go:currentpackage:: hcl + +.. go:type:: Expression + + Represents a unevaluated single expression. + + .. go:function:: func (e Expression) Value(ctx *EvalContext) (cty.Value, Diagnostics) + + Evaluates the receiving expression in the given evaluation context. The + result is a :go:type:`cty.Value` representing the result value, along + with any diagnostics that were raised during evaluation. + + If the diagnostics contains errors, the value may be incomplete or + invalid and should either be discarded altogether or used with care for + analysis. + + .. go:function:: func (e Expression) Variables() []Traversal + + Returns information about any nested expressions that access variables + from the *global* evaluation context. Does not include references to + temporary local variables, such as those generated by a + "``for`` expression". + + .. go:function:: func (e Expression) Range() Range + + Returns the source range for the entire expression. This can be useful + when generating application-specific diagnostic messages, such as + value validation errors. + + .. go:function:: func (e Expression) StartRange() Range + + Similar to ``Range``, but if the expression is complex, such as a tuple + or object constructor, may indicate only the opening tokens for the + construct to avoid creating an overwhelming source code snippet. + + This should be used in diagnostic messages only in situations where the + error is clearly with the construct itself and not with the overall + expression. For example, a type error indicating that a tuple was not + expected might use ``StartRange`` to draw attention to the beginning + of a tuple constructor, without highlighting the entire expression. + +Method ``Value`` is the primary API for expressions, and takes the same kind +of evaluation context object described in :ref:`go-expression-eval`. + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "name": cty.StringVal("Ermintrude"), + "age": cty.NumberIntVal(32), + }, + } + val, moreDiags := expr.Value(ctx) + diags = append(diags, moreDiags...) diff --git a/guide/go_diagnostics.rst b/guide/go_diagnostics.rst new file mode 100644 index 0000000..a948542 --- /dev/null +++ b/guide/go_diagnostics.rst @@ -0,0 +1,97 @@ +.. _go-diagnostics: + +Diagnostic Messages +=================== + +An important concern for any machine language intended for human authoring is +to produce good error messages when the input is somehow invalid, or has +other problems. + +HCL uses *diagnostics* to describe problems in an end-user-oriented manner, +such that the calling application can render helpful error or warning messages. +The word "diagnostic" is a general term that covers both errors and warnings, +where errors are problems that prevent complete processing while warnings are +possible concerns that do not block processing. + +HCL deviates from usual Go API practice by returning its own ``hcl.Diagnostics`` +type, instead of Go's own ``error`` type. This allows functions to return +warnings without accompanying errors while not violating the usual expectation +that the absense of errors is indicated by a nil ``error``. + +In order to easily accumulate and return multiple diagnostics at once, the +usual pattern for functions returning diagnostics is to gather them in a +local variable and then return it at the end of the function, or possibly +earlier if the function cannot continue due to the problems. + +.. code-block:: go + + func returningDiagnosticsExample() hcl.Diagnostics { + var diags hcl.Diagnostics + + // ... + + // Call a function that may itself produce diagnostics. + f, moreDiags := parser.LoadHCLFile("example.conf") + // always append, in case warnings are present + diags = append(diags, moreDiags...) + if diags.HasErrors() { + // If we can't safely continue in the presence of errors here, we + // can optionally return early. + return diags + } + + // ... + + return diags + } + +A common variant of the above pattern is calling another diagnostics-generating +function in a loop, using ``continue`` to begin the next iteration when errors +are detected, but still completing all iterations and returning the union of +all of the problems encountered along the way. + +In :ref:`go-parsing`, we saw that the parser can generate diagnostics which +are related to syntax problems within the loaded file. Further steps to decode +content from the loaded file can also generate diagnostics related to *semantic* +problems within the file, such as invalid expressions or type mismatches, and +so a program using HCL will generally need to accumulate diagnostics across +these various steps and then render them in the application UI somehow. + +Rendering Diagnostics in the UI +------------------------------- + +The best way to render diagnostics to an end-user will depend a lot on the +type of application: they might be printed into a terminal, written into a +log for later review, or even shown in a GUI. + +HCL leaves the responsibility for rendering diagnostics to the calling +application, but since rendering to a terminal is a common case for command-line +tools, the `hcl` package contains a default implementation of this in the +form of a "diagnostic text writer": + +.. code-block:: go + + wr := hcl.NewDiagnosticTextWriter( + os.Stdout, // writer to send messages to + parser.Files(), // the parser's file cache, for source snippets + 78, // wrapping width + true, // generate colored/highlighted output + ) + wr.WriteDiagnostics(diags) + +This default implementation of diagnostic rendering includes relevant lines +of source code for context, like this: + +:: + + Error: Unsupported block type + + on example.tf line 4, in resource "aws_instance" "example": + 2: provisionr "local-exec" { + + Blocks of type "provisionr" are not expected here. Did you mean "provisioner"? + +If the "color" flag is enabled, the severity will be additionally indicated by +a text color and the relevant portion of the source code snippet will be +underlined to draw further attention. + diff --git a/guide/go_expression_eval.rst b/guide/go_expression_eval.rst new file mode 100644 index 0000000..f6ed0b6 --- /dev/null +++ b/guide/go_expression_eval.rst @@ -0,0 +1,149 @@ +.. _go-expression-eval: + +Expression Evaluation +===================== + +Each argument attribute in a configuration file is interpreted as an +expression. In the HCL native syntax, certain basic expression functionality +is always available, such as arithmetic and template strings, and the calling +application can extend this by making available specific variables and/or +functions via an *evaluation context*. + +We saw in :ref:`go-decoding-gohcl` and :ref:`go-decoding-hcldec` some basic +examples of populating an evaluation context to make a variable available. +This section will look more closely at the ``hcl.EvalContext`` type and how +HCL expression evaluation behaves in different cases. + +This section does not discuss in detail the expression syntax itself. For more +information on that, see the HCL Native Syntax specification. + +.. go:currentpackage:: hcl + +.. go:type:: EvalContext + + ``hcl.EvalContext`` is the type used to describe the variables and functions + available during expression evaluation, if any. Its usage is described in + the following sections. + +Defining Variables +------------------ + +As we saw in :ref:`go-decoding-hcldec`, HCL represents values using an +underlying library called :go:pkg:`cty`. When defining variables, their values +must be given as :go:type:`cty.Value` values. + +A full description of the types and value constructors in :go:pkg:`cty` is +in `the reference documentation `_. +Variables in HCL are defined by assigning values into a map from string names +to :go:type:`cty.Value`: + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "name": cty.StringVal("Ermintrude"), + "age": cty.NumberIntVal(32), + }, + } + +If this evaluation context were passed to one of the evaluation functions we +saw in previous sections, the user would be able to refer to these variable +names in any argument expression appearing in the evaluated portion of +configuration: + +.. code-block:: hcl + + message = "${name} is ${age} ${age == 1 ? "year" : "years"} old!" + +If you place ``cty``'s *object* values in the evaluation context, then their +attributes can be referenced using the HCL attribute syntax, allowing for more +complex structures: + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "path": cty.ObjectVal(map[string]cty.Value{ + "root": cty.StringVal(rootDir), + "module": cty.StringVal(moduleDir), + "current": cty.StringVal(currentDir), + }), + }, + } + +.. code-block:: hcl + + source_file = "${path.module}/foo.txt" + +.. _go-expression-funcs: + +Defining Functions +------------------ + +Custom functions can be defined by you application to allow users of its +language to transform data in application-specific ways. The underlying +function mechanism is also provided by :go:pkg:`cty`, allowing you to define +the arguments a given function expects, what value type it will return for +given argument types, etc. The full functions model is described in the +``cty`` documentation section +`Functions System `_. + +There are `a number of "standard library" functions `_ +available in a ``stdlib`` package within the :go:pkg:`cty` repository, avoiding +the need for each application to re-implement basic functions for string +manipulation, list manipulation, etc. It also includes function-shaped versions +of several operations that are native operators in HCL, which should generally +*not* be exposed as functions in HCL-based configurationf formats to avoid user +confusion. + +You can define functions in the ``Functions`` field of :go:type:`hcl.EvalContext`: + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "name": cty.StringVal("Ermintrude"), + }, + Functions: map[string]function.Function{ + "upper": stdlib.UpperFunc, + "lower": stdlib.LowerFunc, + "min": stdlib.MinFunc, + "max": stdlib.MaxFunc, + "strlen": stdlib.StrlenFunc, + "substr": stdlib.SubstrFunc, + }, + } + +If this evaluation context were passed to one of the evaluation functions we +saw in previous sections, the user would be able to call any of these functions +in any argument expression appearing in the evaluated portion of configuration: + +.. code-block:: hcl + + message = "HELLO, ${upper(name)}!" + +Expression Evaluation Modes +--------------------------- + +HCL uses a different expression evaluation mode depending on the evaluation +context provided. In HCL native syntax, evaluation modes are used to provide +more relevant error messages. In JSON syntax, which embeds the native +expression syntax in strings using "template" syntax, the evaluation mode +determines whether strings are evaluated as templates at all. + +If the given :go:type:`hcl.EvalContext` is ``nil``, native syntax expressions +will react to users attempting to refer to variables or functions by producing +errors indicating that these features are not available at all, rather than +by saying that the specific variable or function does not exist. JSON syntax +strings will not be evaluated as templates *at all* in this mode, making them +function as literal strings. + +If the evaluation context is non-``nil`` but either ``Variables`` or +``Functions`` within it is ``nil``, native syntax will similarly produce +"not supported" error messages. JSON syntax strings *will* parse templates +in this case, but can also generate "not supported" messages if e.g. the +user accesses a variable when the variables map is ``nil``. + +If neither map is ``nil``, HCL assumes that both variables and functions are +supported and will instead produce error messages stating that the specific +variable or function accessed by the user is not defined. diff --git a/guide/go_parsing.rst b/guide/go_parsing.rst new file mode 100644 index 0000000..064bdee --- /dev/null +++ b/guide/go_parsing.rst @@ -0,0 +1,64 @@ +.. _go-parsing: + +Parsing HCL Input +================= + +The first step in processing HCL input provided by a user is to parse it. +Parsing turns the raw bytes from an input file into a higher-level +representation of the arguments and blocks, ready to be *decoded* into an +application-specific form. + +The main entry point into HCL parsing is :go:pkg:`hclparse`, which provides +:go:type:`hclparse.Parser`: + +.. code-block:: go + + parser := hclparse.NewParser() + f, diags := parser.ParseHCLFile("server.conf") + +Variable ``f`` is then a pointer to an :go:type:`hcl.File`, which is an +opaque abstract representation of the file, ready to be decoded. + +Variable ``diags`` describes any errors or warnings that were encountered +during processing; HCL conventionally uses this in place of the usual ``error`` +return value in Go, to allow returning a mixture of multiple errors and +warnings together with enough information to present good error messages to the +user. We'll cover this in more detail in the next section, +:ref:`go-diagnostics`. + +.. go:package:: hclparse + +Package ``hclparse`` +-------------------- + +.. go:type:: Parser + + .. go:function:: func NewParser() *Parser + + Constructs a new parser object. Each parser contains a cache of files + that have already been read, so repeated calls to load the same file + will return the same object. + + .. go:function:: func (*Parser) ParseHCL(src []byte, filename string) (*hcl.File, hcl.Diagnostics) + + Parse the given source code as HCL native syntax, saving the result into + the parser's file cache under the given filename. + + .. go:function:: func (*Parser) ParseHCLFile(filename string) (*hcl.File, hcl.Diagnostics) + + Parse the contents of the given file as HCL native syntax. This is a + convenience wrapper around ParseHCL that first reads the file into memory. + + .. go:function:: func (*Parser) ParseJSON(src []byte, filename string) (*hcl.File, hcl.Diagnostics) + + Parse the given source code as JSON syntax, saving the result into + the parser's file cache under the given filename. + + .. go:function:: func (*Parser) ParseJSONFile(filename string) (*hcl.File, hcl.Diagnostics) + + Parse the contents of the given file as JSON syntax. This is a + convenience wrapper around ParseJSON that first reads the file into memory. + +The above list just highlights the main functions in this package. +For full documentation, see +`the hclparse godoc `_. diff --git a/guide/go_patterns.rst b/guide/go_patterns.rst new file mode 100644 index 0000000..0c70496 --- /dev/null +++ b/guide/go_patterns.rst @@ -0,0 +1,315 @@ +Design Patterns for Complex Systems +=================================== + +In previous sections we've seen an overview of some different ways an +application can decode a language its has defined in terms of the HCL grammar. +For many applications, those mechanisms are sufficient. However, there are +some more complex situations that can benefit from some additional techniques. +This section lists a few of these situations and ways to use the HCL API to +accommodate them. + +.. _go-interdep-blocks: + +Interdependent Blocks +--------------------- + +In some configuration languages, the variables available for use in one +configuration block depend on values defined in other blocks. + +For example, in Terraform many of the top-level constructs are also implicitly +definitions of values that are available for use in expressions elsewhere: + +.. code-block:: hcl + + variable "network_numbers" { + type = list(number) + } + + variable "base_network_addr" { + type = string + default = "10.0.0.0/8" + } + + locals { + network_blocks = { + for x in var.number: + x => cidrsubnet(var.base_network_addr, 8, x) + } + } + + resource "cloud_subnet" "example" { + for_each = local.network_blocks + + cidr_block = each.value + } + + output "subnet_ids" { + value = cloud_subnet.example[*].id + } + +In this example, the `variable "network_numbers"` block makes +``var.base_network_addr`` available to expressions, the +``resource "cloud_subnet" "example"`` block makes ``cloud_subnet.example`` +available, etc. + +Terraform achieves this by decoding the top-level structure in isolation to +start. You can do this either using the low-level API or using :go:pkg:`gohcl` +with :go:type:`hcl.Body` fields tagged as "remain". + +Once you have a separate body for each top-level block, you can inspect each +of the attribute expressions inside using the ``Variables`` method on +:go:type:`hcl.Expression`, or the ``Variables`` function from package +:go:pkg:`hcldec` if you will eventually use its higher-level API to decode as +Terraform does. + +The detected variable references can then be used to construct a dependency +graph between the blocks, and then perform a +`topological sort `_ to +determine the correct order to evaluate each block's contents so that values +will always be available before they are needed. + +Since :go:pkg:`cty` values are immutable, it is not convenient to directly +change values in a :go:type:`hcl.EvalContext` during this gradual evaluation, +so instead construct a specialized data structure that has a separate value +per object and construct an evaluation context from that each time a new +value becomes available. + +Using :go:pkg:`hcldec` to evaluate block bodies is particularly convenient in +this scenario because it produces :go:type:`cty.Value` results which can then +just be directly incorporated into the evaluation context. + +Distributed Systems +------------------- + +Distributed systems cause a number of extra challenges, and configuration +management is rarely the worst of these. However, there are some specific +considerations for using HCL-based configuration in distributed systems. + +For the sake of this section, we are concerned with distributed systems where +at least two separate components both depend on the content of HCL-based +configuration files. Real-world examples include the following: + +* **HashiCorp Nomad** loads configuration (job specifications) in its servers + but also needs these results in its clients and in its various driver plugins. + +* **HashiCorp Terraform** parses configuration in Terraform Core but can write + a partially-evaluated execution plan to disk and continue evaluation in a + separate process later. It must also pass configuration values into provider + plugins. + +Broadly speaking, there are two approaches to allowing configuration to be +accessed in multiple subsystems, which the following subsections will discuss +separately. + +Ahead-of-time Evaluation +^^^^^^^^^^^^^^^^^^^^^^^^ + +Ahead-of-time evaluation is the simplest path, with the configuration files +being entirely evaluated on entry to the system, and then only the resulting +*constant values* being passed between subsystems. + +This approach is relatively straightforward because the resulting +:go:type:`cty.Value` results can be losslessly serialized as either JSON or +msgpack as long as all system components agree on the expected value types. +Aside from passing these values around "on the wire", parsing and decoding of +configuration proceeds as normal. + +Both Nomad and Terraform use this approach for interacting with *plugins*, +because the plugins themselves are written by various different teams that do +not coordinate closely, and so doing all expression evaluation in the core +subsystems ensures consistency between plugins and simplifies plugin development. + +In both applications, the plugin is expected to describe (using an +application-specific protocol) the schema it expects for each element of +configuration it is responsible for, allowing the core subsystems to perform +decoding on the plugin's behalf and pass a value that is guaranteed to conform +to the schema. + +Gradual Evaluation +^^^^^^^^^^^^^^^^^^ + +Although ahead-of-time evaluation is relatively straightforward, it has the +significant disadvantage that all data available for access via variables or +functions must be known by whichever subsystem performs that initial +evaluation. + +For example, in Terraform, the "plan" subcommand is responsible for evaluating +the configuration and presenting to the user an execution plan for approval, but +certain values in that plan cannot be determined until the plan is already +being applied, since the specific values used depend on remote API decisions +such as the allocation of opaque id strings for objects. + +In Terraform's case, both the creation of the plan and the eventual apply +of that plan *both* entail evaluating configuration, with the apply step +having a more complete set of input values and thus producing a more complete +result. However, this means that Terraform must somehow make the expressions +from the original input configuration available to the separate process that +applies the generated plan. + +Good usability requires error and warning messages that are able to refer back +to specific sections of the input configuration as context for the reported +problem, and the best way to achieve this in a distributed system doing +gradual evaluation is to send the configuration *source code* between +subsystems. This is generally the most compact representation that retains +source location information, and will avoid any inconsistency caused by +introducing another intermediate serialization. + +In Terraform's, for example, the serialized plan incorporates both the data +structure describing the partial evaluation results from the plan phase and +the original configuration files that produced those results, which can then +be re-evalauated during the apply step. + +In a gradual evaluation scenario, the application should verify correctness of +the input configuration as completely as possible at each state. To help with +this, :go:pkg:`cty` has the concept of +`unknown values `_, +which can stand in for values the application does not yet know while still +retaining correct type information. HCL expression evaluation reacts to unknown +values by performing type checking but then returning another unknown value, +causing the unknowns to propagate through expressions automatically. + +.. code-block:: go + + ctx := &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "name": cty.UnknownVal(cty.String), + "age": cty.UnknownVal(cty.Number), + }, + } + val, moreDiags := expr.Value(ctx) + diags = append(diags, moreDiags...) + +Each time an expression is re-evaluated with additional information, fewer of +the input values will be unknown and thus more of the result will be known. +Eventually the application should evaluate the expressions with no unknown +values at all, which then guarantees that the result will also be wholly-known. + +Static References, Calls, Lists, and Maps +----------------------------------------- + +In most cases, we care more about the final result value of an expression than +how that value was obtained. A particular list argument, for example, might +be defined by the user via a tuple constructor, by a `for` expression, or by +assigning the value of a variable that has a suitable list type. + +In some special cases, the structure of the expression is more important than +the result value, or an expression may not *have* a reasonable result value. +For example, in Terraform there are a few arguments that call for the user +to name another object by reference, rather than provide an object value: + +.. code-block:: hcl + + resource "cloud_network" "example" { + # ... + } + + resource "cloud_subnet" "example" { + cidr_block = "10.1.2.0/24" + + depends_on = [ + cloud_network.example, + ] + } + +The ``depends_on`` argument in the second ``resource`` block *appears* as an +expression that would construct a single-element tuple containing an object +representation of the first resource block. However, Terraform uses this +expression to construct its dependency graph, and so it needs to see +specifically that this expression refers to ``cloud_network.example``, rather +than determine a result value for it. + +HCL offers a number of "static analysis" functions to help with this sort of +situation. These all live in the :go:pkg:`hcl` package, and each one imposes +a particular requirement on the syntax tree of the expression it is given, +and returns a result derived from that if the expression conforms to that +requirement. + +.. go:currentpackage:: hcl + +.. go:function:: func ExprAsKeyword(expr Expression) string + + This function attempts to interpret the given expression as a single keyword, + returning that keyword as a string if possible. + + A "keyword" for the purposes of this function is an expression that can be + understood as a valid single identifier. For example, the simple variable + reference ``foo`` can be interpreted as a keyword, while ``foo.bar`` + cannot. + + As a special case, the language-level keywords ``true``, ``false``, and + ``null`` are also considered to be valid keywords, allowing the calling + application to disregard their usual meaning. + + If the given expression cannot be reduced to a single keyword, the result + is an empty string. Since an empty string is never a valid keyword, this + result unambiguously signals failure. + +.. go:function:: func AbsTraversalForExpr(expr Expression) (Traversal, Diagnostics) + + This is a generalization of ``ExprAsKeyword`` that will accept anything that + can be interpreted as a *traversal*, which is a variable name followed by + zero or more attribute access or index operators with constant operands. + + For example, all of ``foo``, ``foo.bar`` and ``foo[0]`` are valid + traversals, but ``foo[bar]`` is not, because the ``bar`` index is not + constant. + + This is the function that Terraform uses to interpret the items within the + ``depends_on`` sequence in our example above. + + As with ``ExprAsKeyword``, this function has a special case that the + keywords ``true``, ``false``, and ``null`` will be accepted as if they were + variable names by this function, allowing ``null.foo`` to be interpreted + as a traversal even though it would be invalid if evaluated. + + If error diagnostics are returned, the traversal result is invalid and + should not be used. + +.. go:function:: func RelTraversalForExpr(expr Expression) (Traversal, Diagnostics) + + This is very similar to ``AbsTraversalForExpr``, but the result is a + *relative* traversal, which is one whose first name is considered to be + an attribute of some other (implied) object. + + The processing rules are identical to ``AbsTraversalForExpr``, with the + only exception being that the first element of the returned traversal is + marked as being an attribute, rather than as a root variable. + +.. go:function:: func ExprList(expr Expression) ([]Expression, Diagnostics) + + This function requires that the given expression be a tuple constructor, + and if so returns a slice of the element expressions in that constructor. + Applications can then perform further static analysis on these, or evaluate + them as normal. + + If error diagnostics are returned, the result is invalid and should not be + used. + + This is the fucntion that Terraform uses to interpret the expression + assigned to ``depends_on`` in our example above, then in turn using + ``AbsTraversalForExpr`` on each enclosed expression. + +.. go:function:: func ExprMap(expr Expression) ([]KeyValuePair, Diagnostics) + + This function requires that the given expression be an object constructor, + and if so returns a slice of the element key/value pairs in that constructor. + Applications can then perform further static analysis on these, or evaluate + them as normal. + + If error diagnostics are returned, the result is invalid and should not be + used. + +.. go:function:: func ExprCall(expr Expression) (*StaticCall, Diagnostics) + + This function requires that the given expression be a function call, and + if so returns an object describing the name of the called function and + expression objects representing the call arguments. + + If error diagnostics are returned, the result is invalid and should not be + used. + +The ``Variables`` method on :go:type:`hcl.Expression` is also considered to be +a "static analysis" helper, but is built in as a fundamental feature because +analysis of referenced variables is often important for static validation and +for implementing interdependent blocks as we saw in the section above. + diff --git a/guide/index.rst b/guide/index.rst new file mode 100644 index 0000000..67ab2ad --- /dev/null +++ b/guide/index.rst @@ -0,0 +1,35 @@ +HCL Configuration Language +========================== + +HCL is a toolkit for creating structured configuration languages that are both +human- and machine-friendly, for use with command-line tools, servers, etc. + +HCL has both a native syntax, intended to be pleasant to read and write for +humans, and a JSON-based variant that is easier for machines to generate and +parse. The native syntax is inspired by libucl_, `nginx configuration`_, and +others. + +It includes an expression syntax that allows basic inline computation and, with +support from the calling application, use of variables and functions for more +dynamic configuration languages. + +HCL provides a set of constructs that can be used by a calling application to +construct a configuration language. The application defines which argument +names and nested block types are expected, and HCL parses the configuration +file, verifies that it conforms to the expected structure, and returns +high-level objects that the application can use for further processing. + +At present, HCL is primarily intended for use in applications written in Go_, +via its library API. + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + intro + go + language_design + +.. _libucl: https://github.com/vstakhov/libucl +.. _`nginx configuration`: http://nginx.org/en/docs/beginners_guide.html#conf_structure +.. _Go: https://golang.org/ diff --git a/guide/intro.rst b/guide/intro.rst new file mode 100644 index 0000000..d089a11 --- /dev/null +++ b/guide/intro.rst @@ -0,0 +1,108 @@ +.. _intro: + +Introduction to HCL +=================== + +HCL-based configuration is built from two main constructs: arguments and +blocks. The following is an example of a configuration language for a +hypothetical application: + +.. code-block:: hcl + + io_mode = "async" + + service "http" "web_proxy" { + listen_addr = "127.0.0.1:8080" + + process "main" { + command = ["/usr/local/bin/awesome-app", "server"] + } + + process "mgmt" { + command = ["/usr/local/bin/awesome-app", "mgmt"] + } + } + +In the above example, ``io_mode`` is a top-level argument, while ``service`` +introduces a block. Within the body of a block, further arguments and nested +blocks are allowed. A block type may also expect a number of *labels*, which +are the quoted names following the ``service`` keyword in the above example. + +The specific keywords ``io_mode``, ``service``, ``process``, etc here are +application-defined. HCL provides the general block structure syntax, and +can validate and decode configuration based on the application's provided +schema. + +HCL is a structured configuration language rather than a data structure +serialization language. This means that unlike languages such as JSON, YAML, +or TOML, HCL is always decoded using an application-defined schema. + +However, HCL does have a JSON-based alternative syntax, which allows the same +structure above to be generated using a standard JSON serializer when users +wish to generate configuration programmatically rather than hand-write it: + +.. code-block:: json + + { + "io_mode": "async", + "service": { + "http": { + "web_proxy": { + "listen_addr": "127.0.0.1:8080", + "process": { + "main": { + "command": ["/usr/local/bin/awesome-app", "server"] + }, + "mgmt": { + "command": ["/usr/local/bin/awesome-app", "mgmt"] + }, + } + } + } + } + } + +The calling application can choose which syntaxes to support. JSON syntax may +not be important or desirable for certain applications, but it is available for +applications that need it. The schema provided by the calling application +allows JSON input to be properly decoded even though JSON syntax is ambiguous +in various ways, such as whether a JSON object is representing a nested block +or an object expression. + +The collection of arguments and blocks at a particular nesting level is called +a *body*. A file always has a root body containing the top-level elements, +and each block also has its own body representing the elements within it. + +The term "attribute" can also be used to refer to what we've called an +"argument" so far. The term "attribute" is also used for the fields of an +object value in argument expressions, and so "argument" is used to refer +specifically to the type of attribute that appears directly within a body. + +The above examples show the general "texture" of HCL-based configuration. The +full details of the syntax are covered in the language specifications. + +.. todo:: Once the language specification documents have settled into a + final location, link them from above. + +Argument Expressions +-------------------- + +The value of an argument can be a literal value shown above, or it may be an +expression to allow arithmetic, deriving one value from another, etc. + +.. code-block:: hcl + + listen_addr = env.LISTEN_ADDR + +Built-in arithmetic and comparison operators are automatically available in all +HCL-based configuration languages. A calling application may optionally +provide variables that users can reference, like ``env`` in the above example, +and custom functions to transform values in application-specific ways. + +Full details of the expression syntax are in the HCL native syntax +specification. Since JSON does not have an expression syntax, JSON-based +configuration files use the native syntax expression language embedded inside +JSON strings. + +.. todo:: Once the language specification documents have settled into a + final location, link to the native syntax specification from above. diff --git a/guide/language_design.rst b/guide/language_design.rst new file mode 100644 index 0000000..880ac7d --- /dev/null +++ b/guide/language_design.rst @@ -0,0 +1,285 @@ +Configuration Language Design +============================= + +In this section we will cover some conventions for HCL-based configuration +languages that can help make them feel consistent with other HCL-based +languages, and make the best use of HCL's building blocks. + +HCL's native and JSON syntaxes both define a mapping from input bytes to a +higher-level information model. In designing a configuration language based on +HCL, your building blocks are the components in that information model: +blocks, arguments, and expressions. + +Each calling application of HCL, then, effectively defines its own language. +Just as Atom and RSS are higher-level languages built on XML, HashiCorp +Terraform has a higher-level language built on HCL, while HashiCorp Nomad has +its own distinct language that is *also* built on HCL. + +From an end-user perspective, these are distinct languages but have a common +underlying texture. Users of both are therefore likely to bring some +expectations from one to the other, and so this section is an attempt to +codify some of these shared expectations to reduce user surprise. + +These are subjective guidelines however, and so applications may choose to +ignore them entirely or ignore them in certain specialized cases. An +application providing a configuration language for a pre-existing system, for +example, may choose to eschew the identifier naming conventions in this section +in order to exactly match the existing names in that underlying system. + +Language Keywords and Identifiers +--------------------------------- + +Much of the work in defining an HCL-based language is in selecting good names +for arguments, block types, variables, and functions. + +The standard for naming in HCL is to use all-lowercase identifiers with +underscores separating words, like ``service`` or ``io_mode``. HCL identifiers +do allow uppercase letters and dashes, but this primarily for natural +interfacing with external systems that may have other identifier conventions, +and so these should generally be avoided for the identifiers native to your +own language. + +The distinction between "keywords" and other identifiers is really just a +convention. In your own language documentation, you may use the word "keyword" +to refer to names that are presented as an intrinsic part of your language, +such as important top-level block type names. + +Block type names are usually singular, since each block defines a single +object. Use a plural block name only if the block is serving only as a +namespacing container for a number of other objects. A block with a plural +type name will generally contain only nested blocks, and no arguments of its +own. + +Argument names are also singular unless they expect a collection value, in +which case they should be plural. For example, ``name = "foo"`` but +``subnet_ids = ["abc", "123"]``. + +Function names will generally *not* use underscores and will instead just run +words together, as is common in the C standard library. This is a result of +the fact that several of the standard library functions offered in ``cty`` +(covered in a later section) have names that follow C library function names +like ``substr``. This is not a strong rule, and applications that use longer +names may choose to use underscores for them to improve readability. + +Blocks vs. Object Values +------------------------ + +HCL blocks and argument values of object type have quite a similar appearance +in the native syntax, and are identical in JSON syntax: + +.. code-block:: hcl + + block { + foo = bar + } + + # argument with object constructor expression + argument = { + foo = bar + } + +In spite of this superficial similarity, there are some important differences +between these two forms. + +The most significant difference is that a child block can contain nested blocks +of its own, while an object constructor expression can define only attributes +of the object it is creating. + +The user-facing model for blocks is that they generally form the more "rigid" +structure of the language itself, while argument values can be more free-form. +An application will generally define in its schema and documentation all of +the arguments that are valid for a particular block type, while arguments +accepting object constructors are more appropriate for situations where the +arguments themselves are freely selected by the user, such as when the +expression will be converted by the application to a map type. + +As a less contrived example, consider the ``resource`` block type in Terraform +and its use with a particular resource type ``aws_instance``: + +.. code-block:: hcl + + resource "aws_instance" "example" { + ami = "ami-abc123" + instance_type = "t2.micro" + + tags = { + Name = "example instance" + } + + ebs_block_device { + device_name = "hda1" + volume_size = 8 + volume_type = "standard" + } + } + +The top-level block type ``resource`` is fundamental to Terraform itself and +so an obvious candidate for block syntax: it maps directly onto an object in +Terraform's own domain model. + +Within this block we see a mixture of arguments and nested blocks, all defined +as part of the schema of the ``aws_instance`` resource type. The ``tags`` +map here is specified as an argument because its keys are free-form, chosen +by the user and mapped directly onto a map in the underlying system. +``ebs_block_device`` is specified as a nested block, because it is a separate +domain object within the remote system and has a rigid schema of its own. + +As a special case, block syntax may sometimes be used with free-form keys if +those keys each serve as a separate declaration of some first-class object +in the language. For example, Terraform has a top-level block type ``locals`` +which behaves in this way: + +.. code-block:: hcl + + locals { + instance_type = "t2.micro" + instance_id = aws_instance.example.id + } + +Although the argument names in this block are arbitrarily selected by the +user, each one defines a distinct top-level object. In other words, this +approach is used to create a more ergonomic syntax for defining these simple +single-expression objects, as a pragmatic alternative to more verbose and +redundant declarations using blocks: + +.. code-block:: hcl + + local "instance_type" { + value = "t2.micro" + } + local "instance_id" { + value = aws_instance.example.id + } + +The distinction between domain objects, language constructs and user data will +always be subjective, so the final decision is up to you as the language +designer. + +Standard Functions +------------------ + +HCL itself does not define a common set of functions available in all HCL-based +languages; the built-in language operators give a baseline of functionality +that is always available, but applications are free to define functions as they +see fit. + +With that said, there's a number of generally-useful functions that don't +belong to the domain of any one application: string manipulation, sequence +manipulation, date formatting, JSON serialization and parsing, etc. + +Given the general need such functions serve, it's helpful if a similar set of +functions is available with compatible behavior across multiple HCL-based +languages, assuming the language is for an application where function calls +make sense at all. + +The Go implementation of HCL is built on an underlying type and function system +:go:pkg:`cty`, whose usage was introduced in :ref:`go-expression-funcs`. That +library also has a package of "standard library" functions which we encourage +applications to offer with consistent names and compatible behavior, either by +using the standard implementations directly or offering compatible +implementations under the same name. + +The "standard" functions that new configuration formats should consider +offering are: + +* ``abs(number)`` - returns the absolute (positive) value of the given number. +* ``coalesce(vals...)`` - returns the value of the first argument that isn't null. Useful only in formats where null values may appear. +* ``compact(vals...)`` - returns a new tuple with the non-null values given as arguments, preserving order. +* ``concat(seqs...)`` - builds a tuple value by concatenating together all of the given sequence (list or tuple) arguments. +* ``format(fmt, args...)`` - performs simple string formatting similar to the C library function ``printf``. +* ``hasindex(coll, idx)`` - returns true if the given collection has the given index. ``coll`` may be of list, tuple, map, or object type. +* ``int(number)`` - returns the integer component of the given number, rounding towards zero. +* ``jsondecode(str)`` - interprets the given string as JSON format and return the corresponding decoded value. +* ``jsonencode(val)`` - encodes the given value as a JSON string. +* ``length(coll)`` - returns the length of the given collection. +* ``lower(str)`` - converts the letters in the given string to lowercase, using Unicode case folding rules. +* ``max(numbers...)`` - returns the highest of the given number values. +* ``min(numbers...)`` - returns the lowest of the given number values. +* ``sethas(set, val)`` - returns true only if the given set has the given value as an element. +* ``setintersection(sets...)`` - returns the intersection of the given sets +* ``setsubtract(set1, set2)`` - returns a set with the elements from ``set1`` that are not also in ``set2``. +* ``setsymdiff(sets...)`` - returns the symmetric difference of the given sets. +* ``setunion(sets...)`` - returns the union of the given sets. +* ``strlen(str)`` - returns the length of the given string in Unicode grapheme clusters. +* ``substr(str, offset, length)`` - returns a substring from the given string by splitting it between Unicode grapheme clusters. +* ``timeadd(time, duration)`` - takes a timestamp in RFC3339 format and a possibly-negative duration given as a string like ``"1h"`` (for "one hour") and returns a new RFC3339 timestamp after adding the duration to the given timestamp. +* ``upper(str)`` - converts the letters in the given string to uppercase, using Unicode case folding rules. + +Not all of these functions will make sense in all applications. For example, an +application that doesn't use set types at all would have no reason to provide +the set-manipulation functions here. + +Some languages will not provide functions at all, since they are primarily for +assigning values to arguments and thus do not need nor want any custom +computations of those values. + +Block Results as Expression Variables +------------------------------------- + +In some applications, top-level blocks serve also as declarations of variables +(or of attributes of object variables) available during expression evaluation, +as discussed in :ref:`go-interdep-blocks`. + +In this case, it's most intuitive for the variables map in the evaluation +context to contain an value named after each valid top-level block +type and for these values to be object-typed or map-typed and reflect the +structure implied by block type labels. + +For example, an application may have a top-level ``service`` block type +used like this: + +.. code-block:: hcl + + service "http" "web_proxy" { + listen_addr = "127.0.0.1:8080" + + process "main" { + command = ["/usr/local/bin/awesome-app", "server"] + } + + process "mgmt" { + command = ["/usr/local/bin/awesome-app", "mgmt"] + } + } + +If the result of decoding this block were available for use in expressions +elsewhere in configuration, the above convention would call for it to be +available to expressions as an object at ``service.http.web_proxy``. + +If it the contents of the block itself that are offered to evaluation -- or +a superset object *derived* from the block contents -- then the block arguments +can map directly to object attributes, but it is up to the application to +decide which value type is most appropriate for each block type, since this +depends on how multiple blocks of the same type relate to one another, or if +multiple blocks of that type are even allowed. + +In the above example, an application would probably expose the ``listen_addr`` +argument value as ``service.http.web_proxy.listen_addr``, and may choose to +expose the ``process`` blocks as a map of objects using the labels as keys, +which would allow an expression like +``service.http.web_proxy.service["main"].command``. + +If multiple blocks of a given type do not have a significant order relative to +one another, as seems to be the case with these ``process`` blocks, +representation as a map is often the most intuitive. If the ordering of the +blocks *is* significant then a list may be more appropriate, allowing the use +of HCL's "splat operators" for convenient access to child arguments. However, +there is no one-size-fits-all solution here and language designers must +instead consider the likely usage patterns of each value and select the +value representation that best accommodates those patterns. + +Some applications may choose to offer variables with slightly different names +than the top-level blocks in order to allow for more concise references, such +as abbreviating ``service`` to ``svc`` in the above examples. This should be +done with care since it may make the relationship between the two less obvious, +but this may be a good tradeoff for names that are accessed frequently that +might otherwise hurt the readability of expressions they are embedded in. +Familiarity permits brevity. + +Many applications will not make blocks results available for use in other +expressions at all, in which case they are free to select whichever variable +names make sense for what is being exposed. For example, a format may make +environment variable values available for use in expressions, and may do so +either as top-level variables (if no other variables are needed) or as an +object named ``env``, which can be used as in ``env.HOME``. + diff --git a/guide/make.bat b/guide/make.bat new file mode 100644 index 0000000..08ad4e0 --- /dev/null +++ b/guide/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=HCL + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/guide/requirements.txt b/guide/requirements.txt new file mode 100644 index 0000000..421475a --- /dev/null +++ b/guide/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinxcontrib-golangdomain +sphinx-autoapi