go
/
hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

guide: Guide-type documentation for HCL

This commit is contained in:
Martin Atkins 2018-09-05 08:10:27 -07:00
parent d754d5a269 57c9a676d7
commit e78fa0ded2
16 changed files with 1873 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
guide/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
env/*
_build/*

20
guide/Makefile Normal file
View File

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = HCL
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

157
guide/conf.py Normal file
View File

@ -0,0 +1,157 @@
import subprocess
import os
import os.path
# -- Project information -----------------------------------------------------
project = u'HCL'
copyright = u'2018, HashiCorp'
author = u'HashiCorp'
if 'READTHEDOCS_VERSION' in os.environ:
version_str = os.environ['READTHEDOCS_VERSION']
else:
version_str = subprocess.check_output(['git', 'describe', '--always']).strip()
# The short X.Y version
version = unicode(version_str)
# The full version, including alpha/beta/rc tags
release = unicode(version_str)
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.todo',
'sphinx.ext.githubpages',
'sphinxcontrib.golangdomain',
'sphinx.ext.autodoc',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', 'env']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'HCLdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'HCL.tex', u'HCL Documentation',
u'HashiCorp', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'hcl', u'HCL Documentation',
[author], 1)
]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'HCL', u'HCL Documentation',
author, 'HCL', 'One line description of project.',
'Miscellaneous'),
]
# -- Extension configuration -------------------------------------------------
# -- Options for todo extension ----------------------------------------------
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

31
guide/go.rst Normal file
View File

@ -0,0 +1,31 @@
Using HCL in a Go application
=============================
HCL is itself written in Go_ and currently it is primarily intended for use as
a library within other Go programs.
This section describes a number of different ways HCL can be used to define
and process a configuration language within a Go program. For simple situations,
HCL can decode directly into Go ``struct`` values in a similar way as encoding
packages such as ``encoding/json`` and ``encoding/xml``.
The HCL Go API also offers some alternative approaches however, for processing
languages that may be more complex or that include portions whose expected
structure cannot be determined until runtime.
The following sections give an overview of different ways HCL can be used in
a Go program.
.. toctree::
:maxdepth: 1
:caption: Sub-sections:
go_parsing
go_diagnostics
go_decoding_gohcl
go_decoding_hcldec
go_expression_eval
go_decoding_lowlevel
go_patterns
.. _Go: https://golang.org/

130
guide/go_decoding_gohcl.rst Normal file
View File

@ -0,0 +1,130 @@
.. go:package:: gohcl
.. _go-decoding-gohcl:
Decoding Into Native Go Values
==============================
The most straightforward way to access the content of an HCL file is to
decode into native Go values using ``reflect``, similar to the technique used
by packages like ``encoding/json`` and ``encoding/xml``.
Package ``gohcl`` provides functions for this sort of decoding. Function
``DecodeBody`` attempts to extract values from an HCL *body* and write them
into a Go value given as a pointer:
.. code-block:: go
type ServiceConfig struct {
Type string `hcl:"type,label"`
Name string `hcl:"name,label"`
ListenAddr string `hcl:"listen_addr"`
}
type Config struct {
IOMode string `hcl:"io_mode"`
Services []ServiceConfig `hcl:"service,block"`
}
var c Config
moreDiags := gohcl.DecodeBody(f.Body, nil, &c)
diags = append(diags, moreDiags...)
The above example decodes the *root body* of a file ``f``, presumably loaded
previously using a parser, into the variable ``c``. The field labels within
the struct types imply the schema of the expected language, which is a cut-down
version of the hypothetical language we showed in :ref:`intro`.
The struct field labels consist of two comma-separated values. The first is
the name of the corresponding argument or block type as it will appear in
the input file, and the second is the type of element being named. If the
second value is omitted, it defaults to ``attr``, requesting an attribute.
Nested blocks are represented by a struct or a slice of that struct, and the
special element type ``label`` within that struct declares that each instance
of that block type must be followed by one or more block labels. In the above
example, the ``service`` block type is defined to require two labels, named
``type`` and ``name``. For label fields in particular, the given name is used
only to refer to the particular label in error messages when the wrong number
of labels is used.
By default, all declared attributes and blocks are considered to be required.
An optional value is indicated by making its field have a pointer type, in
which case ``nil`` is written to indicate the absense of the argument.
The sections below discuss some additional decoding use-cases. For full details
on the `gohcl` package, see
`the godoc reference <https://godoc.org/github.com/hashicorp/hcl2/gohcl>`_.
.. _go-decoding-gohcl-evalcontext:
Variables and Functions
-----------------------
By default, arguments given in the configuration may use only literal values
and the built in expression language operators, such as arithmetic.
The second argument to ``gohcl.DecodeBody``, shown as ``nil`` in the previous
example, allows the calling application to additionally offer variables and
functions for use in expressions. Its value is a pointer to an
``hcl.EvalContext``, which will be covered in more detail in the later section
:ref:`go-expression-eval`. For now, a simple example of making the id of the
current process available as a single variable called ``pid``:
.. code-block:: go
type Context struct {
Pid string
}
ctx := gohcl.EvalContext(&Context{
Pid: os.Getpid()
})
var c Config
moreDiags := gohcl.DecodeBody(f.Body, ctx, &c)
diags = append(diags, moreDiags...)
``gohcl.EvalContext`` constructs an expression evaluation context from a Go
struct value, making the fields available as variables and the methods
available as functions, after transforming the field and method names such
that each word (starting with an uppercase letter) is all lowercase and
separated by underscores.
.. code-block:: hcl
name = "example-program (${pid})"
Partial Decoding
----------------
In the examples so far, we've extracted the content from the entire input file
in a single call to ``DecodeBody``. This is sufficient for many simple
situations, but sometimes different parts of the file must be evaluated
separately. For example:
* If different parts of the file must be evaluated with different variables
or functions available.
* If the result of evaluating one part of the file is used to set variables
or functions in another part of the file.
There are several ways to perform partial decoding with ``gohcl``, all of
which involve decoding into HCL's own types, such as ``hcl.Body``.
The most general approach is to declare an additional struct field of type
``hcl.Body``, with the special field tag type ``remain``:
.. code-block:: go
type ServiceConfig struct {
Type string `hcl:"type,label"`
Name string `hcl:"name,label"`
ListenAddr string `hcl:"listen_addr"`
Remain hcl.Body `hcl:",remain"`
}
When a ``remain`` field is present, any element of the input body that is
not matched is retained in a body saved into that field, which can then be
decoded in a later call, potentially with a different evaluation context.
Another option is to decode an attribute into a value of type `hcl.Expression`,
which can then be evaluated separately as described in
:ref:`expression-eval`.

242
guide/go_decoding_hcldec.rst Normal file
View File

@ -0,0 +1,242 @@
.. go:package:: hcldec
.. _go-decoding-hcldec:
Decoding With Dynamic Schema
============================
In section :ref:`go-decoding-gohcl`, we saw the most straightforward way to
access the content from an HCL file, decoding directly into a Go value whose
type is known at application compile time.
For some applications, it is not possible to know the schema of the entire
configuration when the application is built. For example, `HashiCorp Terraform`_
uses HCL as the foundation of its configuration language, but parts of the
configuration are handled by plugins loaded dynamically at runtime, and so
the schemas for these portions cannot be encoded directly in the Terraform
source code.
HCL's ``hcldec`` package offers a different approach to decoding that allows
schemas to be created at runtime, and the result to be decoded into
dynamically-typed data structures.
The sections below are an overview of the main parts of package ``hcldec``.
For full details, see
`the package godoc <https://godoc.org/github.com/hashicorp/hcl2/hcldec>`_.
.. _`HashiCorp Terraform`: https://www.terraform.io/
Decoder Specification
---------------------
Whereas :go:pkg:`gohcl` infers the expected schema by using reflection against
the given value, ``hcldec`` obtains schema through a decoding *specification*,
which is a set of instructions for mapping HCL constructs onto a dynamic
data structure.
The ``hcldec`` package contains a number of different specifications, each
implementing :go:type:`hcldec.Spec` and having a ``Spec`` suffix on its name.
Each spec has two distinct functions:
* Adding zero or more validation constraints on the input configuration file.
* Producing a result value based on some elements from the input file.
The most common pattern is for the top-level spec to be a
:go:type:`hcldec.ObjectSpec` with nested specifications defining either blocks
or attributes, depending on whether the configuration file will be
block-structured or flat.
.. code-block:: go
spec := hcldec.ObjectSpec{
"io_mode": &hcldec.AttrSpec{
Name: "io_mode",
Type: cty.String,
},
"services": &hcldec.BlockMapSpec{
TypeName: "service",
LabelNames: []string{"type", "name"},
Nested: hcldec.ObjectSpec{
"listen_addr": &hcldec.AttrSpec{
Name: "listen_addr",
Type: cty.String,
Required: true,
},
"processes": &hcldec.BlockMapSpec{
TypeName: "service",
LabelNames: []string{"name"},
Nested: hcldec.ObjectSpec{
"command": &hcldec.AttrSpec{
Name: "command",
Type: cty.List(cty.String),
Required: true,
},
},
},
},
},
}
val, moreDiags := hcldec.Decode(f.Body, spec, nil)
diags = append(diags, moreDiags...)
The above specification expects a configuration shaped like our example in
:ref:`intro`, and calls for it to be decoded into a dynamic data structure
that would have the following shape if serialized to JSON:
.. code-block:: JSON
{
"io_mode": "async",
"services": {
"http": {
"web_proxy": {
"listen_addr": "127.0.0.1:8080",
"processes": {
"main": {
"command": ["/usr/local/bin/awesome-app", "server"]
},
"mgmt": {
"command": ["/usr/local/bin/awesome-app", "mgmt"]
}
}
}
}
}
}
.. go:package:: cty
Types and Values With ``cty``
-----------------------------
HCL's expression interpreter is implemented in terms of another library called
:go:pkg:`cty`, which provides a type system which HCL builds on and a robust
representation of dynamic values in that type system. You could think of
:go:pkg:`cty` as being a bit like Go's own :go:pkg:`reflect`, but for the
results of HCL expressions rather than Go programs.
The full details of this system can be found in
`its own repository <https://github.com/zclconf/go-cty>`_, but this section
will cover the most important highlights, because ``hcldec`` specifications
include :go:pkg:`cty` types (as seen in the above example) and its results are
:go:pkg:`cty` values.
``hcldec`` works directly with :go:pkg:`cty` — as opposed to converting values
directly into Go native types — because the functionality of the :go:pkg:`cty`
packages then allows further processing of those values without any loss of
fidelity or range. For example, :go:pkg:`cty` defines a JSON encoding of its
values that can be decoded losslessly as long as both sides agree on the value
type that is expected, which is a useful capability in systems where some sort
of RPC barrier separates the main program from its plugins.
Types are instances of :go:type:`cty.Type`, and are constructed from functions
and variables in :go:pkg:`cty` as shown in the above example, where the string
attributes are typed as ``cty.String``, which is a primitive type, and the list
attribute is typed as ``cty.List(cty.String)``, which constructs a new list
type with string elements.
Values are instances of :go:type:`cty.Value`, and can also be constructed from
functions in :go:pkg:`cty`, using the functions that include ``Val`` in their
names or using the operation methods available on :go:type:`cty.Value`.
In most cases you will eventually want to use the resulting data as native Go
types, to pass it to non-:go:pkg:`cty`-aware code. To do this, see the guides
on
`Converting between types <https://github.com/zclconf/go-cty/blob/master/docs/convert.md>`_
(staying within :go:pkg:`cty`) and
`Converting to and from native Go values <https://github.com/zclconf/go-cty/blob/master/docs/gocty.md>`_.
Partial Decoding
----------------
Because the ``hcldec`` result is always a value, the input is always entirely
processed in a single call, unlike with :go:pkg:`gohcl`.
However, both :go:pkg:`gohcl` and :go:pkg:`hcldec` take :go:type:`hcl.Body` as
the representation of input, and so it is possible and common to mix them both
in the same program.
A common situation is that :go:pkg:`gohcl` is used in the main program to
decode the top level of configuration, which then allows the main program to
determine which plugins need to be loaded to process the leaf portions of
configuration. In this case, the portions that will be interpreted by plugins
are retained as opaque :go:type:`hcl.Body` until the plugins have been loaded,
and then each plugin provides its :go:type:`hcldec.Spec` to allow decoding the
plugin-specific configuration into a :go:type:`cty.Value` which be
transmitted to the plugin for further processing.
In our example from :ref:`intro`, perhaps each of the different service types
is managed by a plugin, and so the main program would decode the block headers
to learn which plugins are needed, but process the block bodies dynamically:
.. code-block:: go
type ServiceConfig struct {
Type string `hcl:"type,label"`
Name string `hcl:"name,label"`
PluginConfig hcl.Body `hcl:",remain"`
}
type Config struct {
IOMode string `hcl:"io_mode"`
Services []ServiceConfig `hcl:"service,block"`
}
var c Config
moreDiags := gohcl.DecodeBody(f.Body, nil, &c)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
// (show diags in the UI)
return
}
for _, sc := range c.Services {
pluginName := block.Type
// Totally-hypothetical plugin manager (not part of HCL)
plugin, err := pluginMgr.GetPlugin(pluginName)
if err != nil {
diags = diags.Append(&hcl.Diagnostic{ /* ... */ })
continue
}
spec := plugin.ConfigSpec() // returns hcldec.Spec
// Decode the block body using the plugin's given specification
configVal, moreDiags := hcldec.Decode(sc.PluginConfig, spec, nil)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
continue
}
// Again, hypothetical API within your application itself, and not
// part of HCL. Perhaps plugin system serializes configVal as JSON
// and sends it over to the plugin.
svc := plugin.NewService(configVal)
serviceMgr.AddService(sc.Name, svc)
}
Variables and Functions
-----------------------
The final argument to ``hcldec.Decode`` is an expression evaluation context,
just as with ``gohcl.DecodeBlock``.
This object can be constructed using
:ref:`the gohcl helper function <go-decoding-gohcl-evalcontext>` as before if desired, but
you can also choose to work directly with :go:type:`hcl.EvalContext` as
discussed in :ref:`go-expression-eval`:
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"pid": cty.NumberIntVal(int64(os.Getpid())),
},
}
val, moreDiags := hcldec.Decode(f.Body, spec, ctx)
diags = append(diags, moreDiags...)
As you can see, this lower-level API also uses :go:pkg:`cty`, so it can be
particularly convenient in situations where the result of dynamically decoding
one block must be available to expressions in another block.

199
guide/go_decoding_lowlevel.rst Normal file
View File

@ -0,0 +1,199 @@
.. _go-decoding-lowlevel:
Advanced Decoding With The Low-level API
========================================
In previous sections we've discussed :go:pkg:`gohcl` and :go:pkg:`hcldec`,
which both deal with decoding of HCL bodies and the expressions within them
using a high-level description of the expected configuration schema.
Both of these packages are implemented in terms of HCL's low-level decoding
interfaces, which we will explore in this section.
HCL decoding in the low-level API has two distinct phases:
* Structural decoding: analyzing the arguments and nested blocks present in a
particular body.
* Expression evaluation: obtaining final values for each argument expression
found during structural decoding.
The low-level API gives the calling application full control over when each
body is decoded and when each expression is evaluated, allowing for more
complex configuration formats where e.g. different variables are available in
different contexts, or perhaps expressions within one block can refer to
values defined in another block.
The low-level API also gives more detailed access to source location
information for decoded elements, and so may be desirable for applications that
do a lot of additional validation of decoded data where more specific source
locations lead to better diagnostic messages.
Since all of the decoding mechanisms work with the same :go:type:`hcl.Body`
type, it is fine and expected to mix them within an application to get access
to the more detailed information where needed while using the higher-level APIs
for the more straightforward portions of a configuration language.
The following subsections will give an overview of the low-level API. For full
details, see `the godoc reference <https://godoc.org/github.com/hashicorp/hcl2/hcl>`_.
Structural Decoding
-------------------
As seen in prior sections, :go:type:`hcl.Body` is an opaque representation of
the arguments and child blocks at a particular nesting level. An HCL file has
a root body containing the top-level elements, and then each nested block has
its own body presenting its own content.
:go:type:`hcl.Body` is a Go interface whose methods serve as the structural
decoding API:
.. go:currentpackage:: hcl
.. go:type:: Body
Represents the structural elements at a particular nesting level.
.. go:function:: func (b Body) Content(schema *BodySchema) (*BodyContent, Diagnostics)
Decode the content from the receiving body using the given schema. The
schema is considered exhaustive of all content within the body, and so
any elements not covered by the schema will generate error diagnostics.
.. go:function:: func (b Body) PartialContent(schema *BodySchema) (*BodyContent, Body, Diagnostics)
Similar to `Content`, but allows for additional arguments and block types
that are not described in the given schema. The additional body return
value is a special body that contains only the *remaining* elements, after
extraction of the ones covered by the schema. This returned body can be
used to decode the remaining content elsewhere in the calling program.
.. go:function:: func (b Body) JustAttributes() (Attributes, Diagnostics)
Decode the content from the receving body in a special *attributes-only*
mode, allowing the calling application to enumerate the arguments given
inside the body without needing to predict them in schema.
When this method is used, a body can be treated somewhat like a map
expression, but it still has a rigid structure where the arguments must
be given directly with no expression evaluation. This is an advantage for
declarations that must themselves be resolved before expression
evaluation is possible.
If the body contains any blocks, error diagnostics are returned. JSON
syntax relies on schema to distinguish arguments from nested blocks, and
so a JSON body in attributes-only mode will treat all JSON object
properties as arguments.
.. go:function:: func (b Body) MissingItemRange() Range
Returns a source range that points to where an absent required item in
the body might be placed. This is a "best effort" sort of thing, required
only to be somewhere inside the receving body, as a way to give source
location information for a "missing required argument" sort of error.
The main content-decoding methods each require a :go:type:`hcl.BodySchema`
object describing the expected content. The fields of this type describe the
expected arguments and nested block types respectively:
.. code-block:: go
schema := &hcl.BodySchema{
Attributes: []hcl.AttributeSchema{
{
Name: "io_mode",
Required: false,
},
},
Blocks: []hcl.BlockHeaderSchema{
{
Type: "service",
LabelNames: []string{"type", "name"},
},
},
}
content, moreDiags := body.Content(schema)
diags = append(diags, moreDiags...)
:go:type:`hcl.BodyContent` is the result of both ``Content`` and
``PartialContent``, giving the actual attributes and nested blocks that were
found. Since arguments are uniquely named within a body and unordered, they
are returned as a map. Nested blocks are ordered and may have many instances
of a given type, so they are returned all together in a single slice for
further interpretation by the caller.
Unlike the two higher-level approaches, the low-level API *always* works only
with one nesting level at a time. Decoding a nested block returns the "header"
for that block, giving its type and label values, but its body remains an
:go:type:`hcl.Body` for later decoding.
Each returned attribute corresponds to one of the arguments in the body, and
it has an :go:type:`hcl.Expression` object that can be used to obtain a value
for the argument during expression evaluation, as described in the next
section.
Expression Evaluation
---------------------
Expression evaluation *in general* has its own section, imaginitively titled
:ref:`go-expression-eval`, so this section will focus only on how it is
achieved in the low-level API.
All expression evaluation in the low-level API starts with an
:go:type:`hcl.Expression` object. This is another interface type, with various
implementations depending on the expression type and the syntax it was parsed
from.
.. go:currentpackage:: hcl
.. go:type:: Expression
Represents a unevaluated single expression.
.. go:function:: func (e Expression) Value(ctx *EvalContext) (cty.Value, Diagnostics)
Evaluates the receiving expression in the given evaluation context. The
result is a :go:type:`cty.Value` representing the result value, along
with any diagnostics that were raised during evaluation.
If the diagnostics contains errors, the value may be incomplete or
invalid and should either be discarded altogether or used with care for
analysis.
.. go:function:: func (e Expression) Variables() []Traversal
Returns information about any nested expressions that access variables
from the *global* evaluation context. Does not include references to
temporary local variables, such as those generated by a
"``for`` expression".
.. go:function:: func (e Expression) Range() Range
Returns the source range for the entire expression. This can be useful
when generating application-specific diagnostic messages, such as
value validation errors.
.. go:function:: func (e Expression) StartRange() Range
Similar to ``Range``, but if the expression is complex, such as a tuple
or object constructor, may indicate only the opening tokens for the
construct to avoid creating an overwhelming source code snippet.
This should be used in diagnostic messages only in situations where the
error is clearly with the construct itself and not with the overall
expression. For example, a type error indicating that a tuple was not
expected might use ``StartRange`` to draw attention to the beginning
of a tuple constructor, without highlighting the entire expression.
Method ``Value`` is the primary API for expressions, and takes the same kind
of evaluation context object described in :ref:`go-expression-eval`.
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"name": cty.StringVal("Ermintrude"),
"age": cty.NumberIntVal(32),
},
}
val, moreDiags := expr.Value(ctx)
diags = append(diags, moreDiags...)

97
guide/go_diagnostics.rst Normal file
View File

@ -0,0 +1,97 @@
.. _go-diagnostics:
Diagnostic Messages
===================
An important concern for any machine language intended for human authoring is
to produce good error messages when the input is somehow invalid, or has
other problems.
HCL uses *diagnostics* to describe problems in an end-user-oriented manner,
such that the calling application can render helpful error or warning messages.
The word "diagnostic" is a general term that covers both errors and warnings,
where errors are problems that prevent complete processing while warnings are
possible concerns that do not block processing.
HCL deviates from usual Go API practice by returning its own ``hcl.Diagnostics``
type, instead of Go's own ``error`` type. This allows functions to return
warnings without accompanying errors while not violating the usual expectation
that the absense of errors is indicated by a nil ``error``.
In order to easily accumulate and return multiple diagnostics at once, the
usual pattern for functions returning diagnostics is to gather them in a
local variable and then return it at the end of the function, or possibly
earlier if the function cannot continue due to the problems.
.. code-block:: go
func returningDiagnosticsExample() hcl.Diagnostics {
var diags hcl.Diagnostics
// ...
// Call a function that may itself produce diagnostics.
f, moreDiags := parser.LoadHCLFile("example.conf")
// always append, in case warnings are present
diags = append(diags, moreDiags...)
if diags.HasErrors() {
// If we can't safely continue in the presence of errors here, we
// can optionally return early.
return diags
}
// ...
return diags
}
A common variant of the above pattern is calling another diagnostics-generating
function in a loop, using ``continue`` to begin the next iteration when errors
are detected, but still completing all iterations and returning the union of
all of the problems encountered along the way.
In :ref:`go-parsing`, we saw that the parser can generate diagnostics which
are related to syntax problems within the loaded file. Further steps to decode
content from the loaded file can also generate diagnostics related to *semantic*
problems within the file, such as invalid expressions or type mismatches, and
so a program using HCL will generally need to accumulate diagnostics across
these various steps and then render them in the application UI somehow.
Rendering Diagnostics in the UI
-------------------------------
The best way to render diagnostics to an end-user will depend a lot on the
type of application: they might be printed into a terminal, written into a
log for later review, or even shown in a GUI.
HCL leaves the responsibility for rendering diagnostics to the calling
application, but since rendering to a terminal is a common case for command-line
tools, the `hcl` package contains a default implementation of this in the
form of a "diagnostic text writer":
.. code-block:: go
wr := hcl.NewDiagnosticTextWriter(
os.Stdout, // writer to send messages to
parser.Files(), // the parser's file cache, for source snippets
78, // wrapping width
true, // generate colored/highlighted output
)
wr.WriteDiagnostics(diags)
This default implementation of diagnostic rendering includes relevant lines
of source code for context, like this:
::
Error: Unsupported block type
on example.tf line 4, in resource "aws_instance" "example":
2: provisionr "local-exec" {
Blocks of type "provisionr" are not expected here. Did you mean "provisioner"?
If the "color" flag is enabled, the severity will be additionally indicated by
a text color and the relevant portion of the source code snippet will be
underlined to draw further attention.

149
guide/go_expression_eval.rst Normal file
View File

@ -0,0 +1,149 @@
.. _go-expression-eval:
Expression Evaluation
=====================
Each argument attribute in a configuration file is interpreted as an
expression. In the HCL native syntax, certain basic expression functionality
is always available, such as arithmetic and template strings, and the calling
application can extend this by making available specific variables and/or
functions via an *evaluation context*.
We saw in :ref:`go-decoding-gohcl` and :ref:`go-decoding-hcldec` some basic
examples of populating an evaluation context to make a variable available.
This section will look more closely at the ``hcl.EvalContext`` type and how
HCL expression evaluation behaves in different cases.
This section does not discuss in detail the expression syntax itself. For more
information on that, see the HCL Native Syntax specification.
.. go:currentpackage:: hcl
.. go:type:: EvalContext
``hcl.EvalContext`` is the type used to describe the variables and functions
available during expression evaluation, if any. Its usage is described in
the following sections.
Defining Variables
------------------
As we saw in :ref:`go-decoding-hcldec`, HCL represents values using an
underlying library called :go:pkg:`cty`. When defining variables, their values
must be given as :go:type:`cty.Value` values.
A full description of the types and value constructors in :go:pkg:`cty` is
in `the reference documentation <https://github.com/zclconf/go-cty/blob/master/docs/types.md>`_.
Variables in HCL are defined by assigning values into a map from string names
to :go:type:`cty.Value`:
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"name": cty.StringVal("Ermintrude"),
"age": cty.NumberIntVal(32),
},
}
If this evaluation context were passed to one of the evaluation functions we
saw in previous sections, the user would be able to refer to these variable
names in any argument expression appearing in the evaluated portion of
configuration:
.. code-block:: hcl
message = "${name} is ${age} ${age == 1 ? "year" : "years"} old!"
If you place ``cty``'s *object* values in the evaluation context, then their
attributes can be referenced using the HCL attribute syntax, allowing for more
complex structures:
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"path": cty.ObjectVal(map[string]cty.Value{
"root": cty.StringVal(rootDir),
"module": cty.StringVal(moduleDir),
"current": cty.StringVal(currentDir),
}),
},
}
.. code-block:: hcl
source_file = "${path.module}/foo.txt"
.. _go-expression-funcs:
Defining Functions
------------------
Custom functions can be defined by you application to allow users of its
language to transform data in application-specific ways. The underlying
function mechanism is also provided by :go:pkg:`cty`, allowing you to define
the arguments a given function expects, what value type it will return for
given argument types, etc. The full functions model is described in the
``cty`` documentation section
`Functions System <https://github.com/zclconf/go-cty/blob/master/docs/functions.md>`_.
There are `a number of "standard library" functions <https://godoc.org/github.com/apparentlymart/go-cty/cty/function/stdlib>`_
available in a ``stdlib`` package within the :go:pkg:`cty` repository, avoiding
the need for each application to re-implement basic functions for string
manipulation, list manipulation, etc. It also includes function-shaped versions
of several operations that are native operators in HCL, which should generally
*not* be exposed as functions in HCL-based configurationf formats to avoid user
confusion.
You can define functions in the ``Functions`` field of :go:type:`hcl.EvalContext`:
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"name": cty.StringVal("Ermintrude"),
},
Functions: map[string]function.Function{
"upper": stdlib.UpperFunc,
"lower": stdlib.LowerFunc,
"min": stdlib.MinFunc,
"max": stdlib.MaxFunc,
"strlen": stdlib.StrlenFunc,
"substr": stdlib.SubstrFunc,
},
}
If this evaluation context were passed to one of the evaluation functions we
saw in previous sections, the user would be able to call any of these functions
in any argument expression appearing in the evaluated portion of configuration:
.. code-block:: hcl
message = "HELLO, ${upper(name)}!"
Expression Evaluation Modes
---------------------------
HCL uses a different expression evaluation mode depending on the evaluation
context provided. In HCL native syntax, evaluation modes are used to provide
more relevant error messages. In JSON syntax, which embeds the native
expression syntax in strings using "template" syntax, the evaluation mode
determines whether strings are evaluated as templates at all.
If the given :go:type:`hcl.EvalContext` is ``nil``, native syntax expressions
will react to users attempting to refer to variables or functions by producing
errors indicating that these features are not available at all, rather than
by saying that the specific variable or function does not exist. JSON syntax
strings will not be evaluated as templates *at all* in this mode, making them
function as literal strings.
If the evaluation context is non-``nil`` but either ``Variables`` or
``Functions`` within it is ``nil``, native syntax will similarly produce
"not supported" error messages. JSON syntax strings *will* parse templates
in this case, but can also generate "not supported" messages if e.g. the
user accesses a variable when the variables map is ``nil``.
If neither map is ``nil``, HCL assumes that both variables and functions are
supported and will instead produce error messages stating that the specific
variable or function accessed by the user is not defined.

64
guide/go_parsing.rst Normal file
View File

@ -0,0 +1,64 @@
.. _go-parsing:
Parsing HCL Input
=================
The first step in processing HCL input provided by a user is to parse it.
Parsing turns the raw bytes from an input file into a higher-level
representation of the arguments and blocks, ready to be *decoded* into an
application-specific form.
The main entry point into HCL parsing is :go:pkg:`hclparse`, which provides
:go:type:`hclparse.Parser`:
.. code-block:: go
parser := hclparse.NewParser()
f, diags := parser.ParseHCLFile("server.conf")
Variable ``f`` is then a pointer to an :go:type:`hcl.File`, which is an
opaque abstract representation of the file, ready to be decoded.
Variable ``diags`` describes any errors or warnings that were encountered
during processing; HCL conventionally uses this in place of the usual ``error``
return value in Go, to allow returning a mixture of multiple errors and
warnings together with enough information to present good error messages to the
user. We'll cover this in more detail in the next section,
:ref:`go-diagnostics`.
.. go:package:: hclparse
Package ``hclparse``
--------------------
.. go:type:: Parser
.. go:function:: func NewParser() *Parser
Constructs a new parser object. Each parser contains a cache of files
that have already been read, so repeated calls to load the same file
will return the same object.
.. go:function:: func (*Parser) ParseHCL(src []byte, filename string) (*hcl.File, hcl.Diagnostics)
Parse the given source code as HCL native syntax, saving the result into
the parser's file cache under the given filename.
.. go:function:: func (*Parser) ParseHCLFile(filename string) (*hcl.File, hcl.Diagnostics)
Parse the contents of the given file as HCL native syntax. This is a
convenience wrapper around ParseHCL that first reads the file into memory.
.. go:function:: func (*Parser) ParseJSON(src []byte, filename string) (*hcl.File, hcl.Diagnostics)
Parse the given source code as JSON syntax, saving the result into
the parser's file cache under the given filename.
.. go:function:: func (*Parser) ParseJSONFile(filename string) (*hcl.File, hcl.Diagnostics)
Parse the contents of the given file as JSON syntax. This is a
convenience wrapper around ParseJSON that first reads the file into memory.
The above list just highlights the main functions in this package.
For full documentation, see
`the hclparse godoc <https://godoc.org/github.com/hashicorp/hcl2/hclparse>`_.

315
guide/go_patterns.rst Normal file
View File

@ -0,0 +1,315 @@
Design Patterns for Complex Systems
===================================
In previous sections we've seen an overview of some different ways an
application can decode a language its has defined in terms of the HCL grammar.
For many applications, those mechanisms are sufficient. However, there are
some more complex situations that can benefit from some additional techniques.
This section lists a few of these situations and ways to use the HCL API to
accommodate them.
.. _go-interdep-blocks:
Interdependent Blocks
---------------------
In some configuration languages, the variables available for use in one
configuration block depend on values defined in other blocks.
For example, in Terraform many of the top-level constructs are also implicitly
definitions of values that are available for use in expressions elsewhere:
.. code-block:: hcl
variable "network_numbers" {
type = list(number)
}
variable "base_network_addr" {
type = string
default = "10.0.0.0/8"
}
locals {
network_blocks = {
for x in var.number:
x => cidrsubnet(var.base_network_addr, 8, x)
}
}
resource "cloud_subnet" "example" {
for_each = local.network_blocks
cidr_block = each.value
}
output "subnet_ids" {
value = cloud_subnet.example[*].id
}
In this example, the `variable "network_numbers"` block makes
``var.base_network_addr`` available to expressions, the
``resource "cloud_subnet" "example"`` block makes ``cloud_subnet.example``
available, etc.
Terraform achieves this by decoding the top-level structure in isolation to
start. You can do this either using the low-level API or using :go:pkg:`gohcl`
with :go:type:`hcl.Body` fields tagged as "remain".
Once you have a separate body for each top-level block, you can inspect each
of the attribute expressions inside using the ``Variables`` method on
:go:type:`hcl.Expression`, or the ``Variables`` function from package
:go:pkg:`hcldec` if you will eventually use its higher-level API to decode as
Terraform does.
The detected variable references can then be used to construct a dependency
graph between the blocks, and then perform a
`topological sort <https://en.wikipedia.org/wiki/Topological_sorting>`_ to
determine the correct order to evaluate each block's contents so that values
will always be available before they are needed.
Since :go:pkg:`cty` values are immutable, it is not convenient to directly
change values in a :go:type:`hcl.EvalContext` during this gradual evaluation,
so instead construct a specialized data structure that has a separate value
per object and construct an evaluation context from that each time a new
value becomes available.
Using :go:pkg:`hcldec` to evaluate block bodies is particularly convenient in
this scenario because it produces :go:type:`cty.Value` results which can then
just be directly incorporated into the evaluation context.
Distributed Systems
-------------------
Distributed systems cause a number of extra challenges, and configuration
management is rarely the worst of these. However, there are some specific
considerations for using HCL-based configuration in distributed systems.
For the sake of this section, we are concerned with distributed systems where
at least two separate components both depend on the content of HCL-based
configuration files. Real-world examples include the following:
* **HashiCorp Nomad** loads configuration (job specifications) in its servers
but also needs these results in its clients and in its various driver plugins.
* **HashiCorp Terraform** parses configuration in Terraform Core but can write
a partially-evaluated execution plan to disk and continue evaluation in a
separate process later. It must also pass configuration values into provider
plugins.
Broadly speaking, there are two approaches to allowing configuration to be
accessed in multiple subsystems, which the following subsections will discuss
separately.
Ahead-of-time Evaluation
^^^^^^^^^^^^^^^^^^^^^^^^
Ahead-of-time evaluation is the simplest path, with the configuration files
being entirely evaluated on entry to the system, and then only the resulting
*constant values* being passed between subsystems.
This approach is relatively straightforward because the resulting
:go:type:`cty.Value` results can be losslessly serialized as either JSON or
msgpack as long as all system components agree on the expected value types.
Aside from passing these values around "on the wire", parsing and decoding of
configuration proceeds as normal.
Both Nomad and Terraform use this approach for interacting with *plugins*,
because the plugins themselves are written by various different teams that do
not coordinate closely, and so doing all expression evaluation in the core
subsystems ensures consistency between plugins and simplifies plugin development.
In both applications, the plugin is expected to describe (using an
application-specific protocol) the schema it expects for each element of
configuration it is responsible for, allowing the core subsystems to perform
decoding on the plugin's behalf and pass a value that is guaranteed to conform
to the schema.
Gradual Evaluation
^^^^^^^^^^^^^^^^^^
Although ahead-of-time evaluation is relatively straightforward, it has the
significant disadvantage that all data available for access via variables or
functions must be known by whichever subsystem performs that initial
evaluation.
For example, in Terraform, the "plan" subcommand is responsible for evaluating
the configuration and presenting to the user an execution plan for approval, but
certain values in that plan cannot be determined until the plan is already
being applied, since the specific values used depend on remote API decisions
such as the allocation of opaque id strings for objects.
In Terraform's case, both the creation of the plan and the eventual apply
of that plan *both* entail evaluating configuration, with the apply step
having a more complete set of input values and thus producing a more complete
result. However, this means that Terraform must somehow make the expressions
from the original input configuration available to the separate process that
applies the generated plan.
Good usability requires error and warning messages that are able to refer back
to specific sections of the input configuration as context for the reported
problem, and the best way to achieve this in a distributed system doing
gradual evaluation is to send the configuration *source code* between
subsystems. This is generally the most compact representation that retains
source location information, and will avoid any inconsistency caused by
introducing another intermediate serialization.
In Terraform's, for example, the serialized plan incorporates both the data
structure describing the partial evaluation results from the plan phase and
the original configuration files that produced those results, which can then
be re-evalauated during the apply step.
In a gradual evaluation scenario, the application should verify correctness of
the input configuration as completely as possible at each state. To help with
this, :go:pkg:`cty` has the concept of
`unknown values <https://github.com/zclconf/go-cty/blob/master/docs/concepts.md#unknown-values-and-the-dynamic-pseudo-type>`_,
which can stand in for values the application does not yet know while still
retaining correct type information. HCL expression evaluation reacts to unknown
values by performing type checking but then returning another unknown value,
causing the unknowns to propagate through expressions automatically.
.. code-block:: go
ctx := &hcl.EvalContext{
Variables: map[string]cty.Value{
"name": cty.UnknownVal(cty.String),
"age": cty.UnknownVal(cty.Number),
},
}
val, moreDiags := expr.Value(ctx)
diags = append(diags, moreDiags...)
Each time an expression is re-evaluated with additional information, fewer of
the input values will be unknown and thus more of the result will be known.
Eventually the application should evaluate the expressions with no unknown
values at all, which then guarantees that the result will also be wholly-known.
Static References, Calls, Lists, and Maps
-----------------------------------------
In most cases, we care more about the final result value of an expression than
how that value was obtained. A particular list argument, for example, might
be defined by the user via a tuple constructor, by a `for` expression, or by
assigning the value of a variable that has a suitable list type.
In some special cases, the structure of the expression is more important than
the result value, or an expression may not *have* a reasonable result value.
For example, in Terraform there are a few arguments that call for the user
to name another object by reference, rather than provide an object value:
.. code-block:: hcl
resource "cloud_network" "example" {
# ...
}
resource "cloud_subnet" "example" {
cidr_block = "10.1.2.0/24"
depends_on = [
cloud_network.example,
]
}
The ``depends_on`` argument in the second ``resource`` block *appears* as an
expression that would construct a single-element tuple containing an object
representation of the first resource block. However, Terraform uses this
expression to construct its dependency graph, and so it needs to see
specifically that this expression refers to ``cloud_network.example``, rather
than determine a result value for it.
HCL offers a number of "static analysis" functions to help with this sort of
situation. These all live in the :go:pkg:`hcl` package, and each one imposes
a particular requirement on the syntax tree of the expression it is given,
and returns a result derived from that if the expression conforms to that
requirement.
.. go:currentpackage:: hcl
.. go:function:: func ExprAsKeyword(expr Expression) string
This function attempts to interpret the given expression as a single keyword,
returning that keyword as a string if possible.
A "keyword" for the purposes of this function is an expression that can be
understood as a valid single identifier. For example, the simple variable
reference ``foo`` can be interpreted as a keyword, while ``foo.bar``
cannot.
As a special case, the language-level keywords ``true``, ``false``, and
``null`` are also considered to be valid keywords, allowing the calling
application to disregard their usual meaning.
If the given expression cannot be reduced to a single keyword, the result
is an empty string. Since an empty string is never a valid keyword, this
result unambiguously signals failure.
.. go:function:: func AbsTraversalForExpr(expr Expression) (Traversal, Diagnostics)
This is a generalization of ``ExprAsKeyword`` that will accept anything that
can be interpreted as a *traversal*, which is a variable name followed by
zero or more attribute access or index operators with constant operands.
For example, all of ``foo``, ``foo.bar`` and ``foo[0]`` are valid
traversals, but ``foo[bar]`` is not, because the ``bar`` index is not
constant.
This is the function that Terraform uses to interpret the items within the
``depends_on`` sequence in our example above.
As with ``ExprAsKeyword``, this function has a special case that the
keywords ``true``, ``false``, and ``null`` will be accepted as if they were
variable names by this function, allowing ``null.foo`` to be interpreted
as a traversal even though it would be invalid if evaluated.
If error diagnostics are returned, the traversal result is invalid and
should not be used.
.. go:function:: func RelTraversalForExpr(expr Expression) (Traversal, Diagnostics)
This is very similar to ``AbsTraversalForExpr``, but the result is a
*relative* traversal, which is one whose first name is considered to be
an attribute of some other (implied) object.
The processing rules are identical to ``AbsTraversalForExpr``, with the
only exception being that the first element of the returned traversal is
marked as being an attribute, rather than as a root variable.
.. go:function:: func ExprList(expr Expression) ([]Expression, Diagnostics)
This function requires that the given expression be a tuple constructor,
and if so returns a slice of the element expressions in that constructor.
Applications can then perform further static analysis on these, or evaluate
them as normal.
If error diagnostics are returned, the result is invalid and should not be
used.
This is the fucntion that Terraform uses to interpret the expression
assigned to ``depends_on`` in our example above, then in turn using
``AbsTraversalForExpr`` on each enclosed expression.
.. go:function:: func ExprMap(expr Expression) ([]KeyValuePair, Diagnostics)
This function requires that the given expression be an object constructor,
and if so returns a slice of the element key/value pairs in that constructor.
Applications can then perform further static analysis on these, or evaluate
them as normal.
If error diagnostics are returned, the result is invalid and should not be
used.
.. go:function:: func ExprCall(expr Expression) (*StaticCall, Diagnostics)
This function requires that the given expression be a function call, and
if so returns an object describing the name of the called function and
expression objects representing the call arguments.
If error diagnostics are returned, the result is invalid and should not be
used.
The ``Variables`` method on :go:type:`hcl.Expression` is also considered to be
a "static analysis" helper, but is built in as a fundamental feature because
analysis of referenced variables is often important for static validation and
for implementing interdependent blocks as we saw in the section above.

35
guide/index.rst Normal file
View File

@ -0,0 +1,35 @@
HCL Configuration Language
==========================
HCL is a toolkit for creating structured configuration languages that are both
human- and machine-friendly, for use with command-line tools, servers, etc.
HCL has both a native syntax, intended to be pleasant to read and write for
humans, and a JSON-based variant that is easier for machines to generate and
parse. The native syntax is inspired by libucl_, `nginx configuration`_, and
others.
It includes an expression syntax that allows basic inline computation and, with
support from the calling application, use of variables and functions for more
dynamic configuration languages.
HCL provides a set of constructs that can be used by a calling application to
construct a configuration language. The application defines which argument
names and nested block types are expected, and HCL parses the configuration
file, verifies that it conforms to the expected structure, and returns
high-level objects that the application can use for further processing.
At present, HCL is primarily intended for use in applications written in Go_,
via its library API.
.. toctree::
:maxdepth: 1
:caption: Contents:
intro
go
language_design
.. _libucl: https://github.com/vstakhov/libucl
.. _`nginx configuration`: http://nginx.org/en/docs/beginners_guide.html#conf_structure
.. _Go: https://golang.org/

108
guide/intro.rst Normal file
View File

@ -0,0 +1,108 @@
.. _intro:
Introduction to HCL
===================
HCL-based configuration is built from two main constructs: arguments and
blocks. The following is an example of a configuration language for a
hypothetical application:
.. code-block:: hcl
io_mode = "async"
service "http" "web_proxy" {
listen_addr = "127.0.0.1:8080"
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "mgmt" {
command = ["/usr/local/bin/awesome-app", "mgmt"]
}
}
In the above example, ``io_mode`` is a top-level argument, while ``service``
introduces a block. Within the body of a block, further arguments and nested
blocks are allowed. A block type may also expect a number of *labels*, which
are the quoted names following the ``service`` keyword in the above example.
The specific keywords ``io_mode``, ``service``, ``process``, etc here are
application-defined. HCL provides the general block structure syntax, and
can validate and decode configuration based on the application's provided
schema.
HCL is a structured configuration language rather than a data structure
serialization language. This means that unlike languages such as JSON, YAML,
or TOML, HCL is always decoded using an application-defined schema.
However, HCL does have a JSON-based alternative syntax, which allows the same
structure above to be generated using a standard JSON serializer when users
wish to generate configuration programmatically rather than hand-write it:
.. code-block:: json
{
"io_mode": "async",
"service": {
"http": {
"web_proxy": {
"listen_addr": "127.0.0.1:8080",
"process": {
"main": {
"command": ["/usr/local/bin/awesome-app", "server"]
},
"mgmt": {
"command": ["/usr/local/bin/awesome-app", "mgmt"]
},
}
}
}
}
}
The calling application can choose which syntaxes to support. JSON syntax may
not be important or desirable for certain applications, but it is available for
applications that need it. The schema provided by the calling application
allows JSON input to be properly decoded even though JSON syntax is ambiguous
in various ways, such as whether a JSON object is representing a nested block
or an object expression.
The collection of arguments and blocks at a particular nesting level is called
a *body*. A file always has a root body containing the top-level elements,
and each block also has its own body representing the elements within it.
The term "attribute" can also be used to refer to what we've called an
"argument" so far. The term "attribute" is also used for the fields of an
object value in argument expressions, and so "argument" is used to refer
specifically to the type of attribute that appears directly within a body.
The above examples show the general "texture" of HCL-based configuration. The
full details of the syntax are covered in the language specifications.
.. todo:: Once the language specification documents have settled into a
final location, link them from above.
Argument Expressions
--------------------
The value of an argument can be a literal value shown above, or it may be an
expression to allow arithmetic, deriving one value from another, etc.
.. code-block:: hcl
listen_addr = env.LISTEN_ADDR
Built-in arithmetic and comparison operators are automatically available in all
HCL-based configuration languages. A calling application may optionally
provide variables that users can reference, like ``env`` in the above example,
and custom functions to transform values in application-specific ways.
Full details of the expression syntax are in the HCL native syntax
specification. Since JSON does not have an expression syntax, JSON-based
configuration files use the native syntax expression language embedded inside
JSON strings.
.. todo:: Once the language specification documents have settled into a
final location, link to the native syntax specification from above.

285
guide/language_design.rst Normal file
View File

@ -0,0 +1,285 @@
Configuration Language Design
=============================
In this section we will cover some conventions for HCL-based configuration
languages that can help make them feel consistent with other HCL-based
languages, and make the best use of HCL's building blocks.
HCL's native and JSON syntaxes both define a mapping from input bytes to a
higher-level information model. In designing a configuration language based on
HCL, your building blocks are the components in that information model:
blocks, arguments, and expressions.
Each calling application of HCL, then, effectively defines its own language.
Just as Atom and RSS are higher-level languages built on XML, HashiCorp
Terraform has a higher-level language built on HCL, while HashiCorp Nomad has
its own distinct language that is *also* built on HCL.
From an end-user perspective, these are distinct languages but have a common
underlying texture. Users of both are therefore likely to bring some
expectations from one to the other, and so this section is an attempt to
codify some of these shared expectations to reduce user surprise.
These are subjective guidelines however, and so applications may choose to
ignore them entirely or ignore them in certain specialized cases. An
application providing a configuration language for a pre-existing system, for
example, may choose to eschew the identifier naming conventions in this section
in order to exactly match the existing names in that underlying system.
Language Keywords and Identifiers
---------------------------------
Much of the work in defining an HCL-based language is in selecting good names
for arguments, block types, variables, and functions.
The standard for naming in HCL is to use all-lowercase identifiers with
underscores separating words, like ``service`` or ``io_mode``. HCL identifiers
do allow uppercase letters and dashes, but this primarily for natural
interfacing with external systems that may have other identifier conventions,
and so these should generally be avoided for the identifiers native to your
own language.
The distinction between "keywords" and other identifiers is really just a
convention. In your own language documentation, you may use the word "keyword"
to refer to names that are presented as an intrinsic part of your language,
such as important top-level block type names.
Block type names are usually singular, since each block defines a single
object. Use a plural block name only if the block is serving only as a
namespacing container for a number of other objects. A block with a plural
type name will generally contain only nested blocks, and no arguments of its
own.
Argument names are also singular unless they expect a collection value, in
which case they should be plural. For example, ``name = "foo"`` but
``subnet_ids = ["abc", "123"]``.
Function names will generally *not* use underscores and will instead just run
words together, as is common in the C standard library. This is a result of
the fact that several of the standard library functions offered in ``cty``
(covered in a later section) have names that follow C library function names
like ``substr``. This is not a strong rule, and applications that use longer
names may choose to use underscores for them to improve readability.
Blocks vs. Object Values
------------------------
HCL blocks and argument values of object type have quite a similar appearance
in the native syntax, and are identical in JSON syntax:
.. code-block:: hcl
block {
foo = bar
}
# argument with object constructor expression
argument = {
foo = bar
}
In spite of this superficial similarity, there are some important differences
between these two forms.
The most significant difference is that a child block can contain nested blocks
of its own, while an object constructor expression can define only attributes
of the object it is creating.
The user-facing model for blocks is that they generally form the more "rigid"
structure of the language itself, while argument values can be more free-form.
An application will generally define in its schema and documentation all of
the arguments that are valid for a particular block type, while arguments
accepting object constructors are more appropriate for situations where the
arguments themselves are freely selected by the user, such as when the
expression will be converted by the application to a map type.
As a less contrived example, consider the ``resource`` block type in Terraform
and its use with a particular resource type ``aws_instance``:
.. code-block:: hcl
resource "aws_instance" "example" {
ami = "ami-abc123"
instance_type = "t2.micro"
tags = {
Name = "example instance"
}
ebs_block_device {
device_name = "hda1"
volume_size = 8
volume_type = "standard"
}
}
The top-level block type ``resource`` is fundamental to Terraform itself and
so an obvious candidate for block syntax: it maps directly onto an object in
Terraform's own domain model.
Within this block we see a mixture of arguments and nested blocks, all defined
as part of the schema of the ``aws_instance`` resource type. The ``tags``
map here is specified as an argument because its keys are free-form, chosen
by the user and mapped directly onto a map in the underlying system.
``ebs_block_device`` is specified as a nested block, because it is a separate
domain object within the remote system and has a rigid schema of its own.
As a special case, block syntax may sometimes be used with free-form keys if
those keys each serve as a separate declaration of some first-class object
in the language. For example, Terraform has a top-level block type ``locals``
which behaves in this way:
.. code-block:: hcl
locals {
instance_type = "t2.micro"
instance_id = aws_instance.example.id
}
Although the argument names in this block are arbitrarily selected by the
user, each one defines a distinct top-level object. In other words, this
approach is used to create a more ergonomic syntax for defining these simple
single-expression objects, as a pragmatic alternative to more verbose and
redundant declarations using blocks:
.. code-block:: hcl
local "instance_type" {
value = "t2.micro"
}
local "instance_id" {
value = aws_instance.example.id
}
The distinction between domain objects, language constructs and user data will
always be subjective, so the final decision is up to you as the language
designer.
Standard Functions
------------------
HCL itself does not define a common set of functions available in all HCL-based
languages; the built-in language operators give a baseline of functionality
that is always available, but applications are free to define functions as they
see fit.
With that said, there's a number of generally-useful functions that don't
belong to the domain of any one application: string manipulation, sequence
manipulation, date formatting, JSON serialization and parsing, etc.
Given the general need such functions serve, it's helpful if a similar set of
functions is available with compatible behavior across multiple HCL-based
languages, assuming the language is for an application where function calls
make sense at all.
The Go implementation of HCL is built on an underlying type and function system
:go:pkg:`cty`, whose usage was introduced in :ref:`go-expression-funcs`. That
library also has a package of "standard library" functions which we encourage
applications to offer with consistent names and compatible behavior, either by
using the standard implementations directly or offering compatible
implementations under the same name.
The "standard" functions that new configuration formats should consider
offering are:
* ``abs(number)`` - returns the absolute (positive) value of the given number.
* ``coalesce(vals...)`` - returns the value of the first argument that isn't null. Useful only in formats where null values may appear.
* ``compact(vals...)`` - returns a new tuple with the non-null values given as arguments, preserving order.
* ``concat(seqs...)`` - builds a tuple value by concatenating together all of the given sequence (list or tuple) arguments.
* ``format(fmt, args...)`` - performs simple string formatting similar to the C library function ``printf``.
* ``hasindex(coll, idx)`` - returns true if the given collection has the given index. ``coll`` may be of list, tuple, map, or object type.
* ``int(number)`` - returns the integer component of the given number, rounding towards zero.
* ``jsondecode(str)`` - interprets the given string as JSON format and return the corresponding decoded value.
* ``jsonencode(val)`` - encodes the given value as a JSON string.
* ``length(coll)`` - returns the length of the given collection.
* ``lower(str)`` - converts the letters in the given string to lowercase, using Unicode case folding rules.
* ``max(numbers...)`` - returns the highest of the given number values.
* ``min(numbers...)`` - returns the lowest of the given number values.
* ``sethas(set, val)`` - returns true only if the given set has the given value as an element.
* ``setintersection(sets...)`` - returns the intersection of the given sets
* ``setsubtract(set1, set2)`` - returns a set with the elements from ``set1`` that are not also in ``set2``.
* ``setsymdiff(sets...)`` - returns the symmetric difference of the given sets.
* ``setunion(sets...)`` - returns the union of the given sets.
* ``strlen(str)`` - returns the length of the given string in Unicode grapheme clusters.
* ``substr(str, offset, length)`` - returns a substring from the given string by splitting it between Unicode grapheme clusters.
* ``timeadd(time, duration)`` - takes a timestamp in RFC3339 format and a possibly-negative duration given as a string like ``"1h"`` (for "one hour") and returns a new RFC3339 timestamp after adding the duration to the given timestamp.
* ``upper(str)`` - converts the letters in the given string to uppercase, using Unicode case folding rules.
Not all of these functions will make sense in all applications. For example, an
application that doesn't use set types at all would have no reason to provide
the set-manipulation functions here.
Some languages will not provide functions at all, since they are primarily for
assigning values to arguments and thus do not need nor want any custom
computations of those values.
Block Results as Expression Variables
-------------------------------------
In some applications, top-level blocks serve also as declarations of variables
(or of attributes of object variables) available during expression evaluation,
as discussed in :ref:`go-interdep-blocks`.
In this case, it's most intuitive for the variables map in the evaluation
context to contain an value named after each valid top-level block
type and for these values to be object-typed or map-typed and reflect the
structure implied by block type labels.
For example, an application may have a top-level ``service`` block type
used like this:
.. code-block:: hcl
service "http" "web_proxy" {
listen_addr = "127.0.0.1:8080"
process "main" {
command = ["/usr/local/bin/awesome-app", "server"]
}
process "mgmt" {
command = ["/usr/local/bin/awesome-app", "mgmt"]
}
}
If the result of decoding this block were available for use in expressions
elsewhere in configuration, the above convention would call for it to be
available to expressions as an object at ``service.http.web_proxy``.
If it the contents of the block itself that are offered to evaluation -- or
a superset object *derived* from the block contents -- then the block arguments
can map directly to object attributes, but it is up to the application to
decide which value type is most appropriate for each block type, since this
depends on how multiple blocks of the same type relate to one another, or if
multiple blocks of that type are even allowed.
In the above example, an application would probably expose the ``listen_addr``
argument value as ``service.http.web_proxy.listen_addr``, and may choose to
expose the ``process`` blocks as a map of objects using the labels as keys,
which would allow an expression like
``service.http.web_proxy.service["main"].command``.
If multiple blocks of a given type do not have a significant order relative to
one another, as seems to be the case with these ``process`` blocks,
representation as a map is often the most intuitive. If the ordering of the
blocks *is* significant then a list may be more appropriate, allowing the use
of HCL's "splat operators" for convenient access to child arguments. However,
there is no one-size-fits-all solution here and language designers must
instead consider the likely usage patterns of each value and select the
value representation that best accommodates those patterns.
Some applications may choose to offer variables with slightly different names
than the top-level blocks in order to allow for more concise references, such
as abbreviating ``service`` to ``svc`` in the above examples. This should be
done with care since it may make the relationship between the two less obvious,
but this may be a good tradeoff for names that are accessed frequently that
might otherwise hurt the readability of expressions they are embedded in.
Familiarity permits brevity.
Many applications will not make blocks results available for use in other
expressions at all, in which case they are free to select whichever variable
names make sense for what is being exposed. For example, a format may make
environment variable values available for use in expressions, and may do so
either as top-level variables (if no other variables are needed) or as an
object named ``env``, which can be used as in ``env.HOME``.

36
guide/make.bat Normal file
View File

@ -0,0 +1,36 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
set SPHINXPROJ=HCL
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd

3
guide/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
sphinx
sphinxcontrib-golangdomain
sphinx-autoapi