guide: Start of HCL usage guide

This is guide-style documentation to introduce the different parts of HCL,
as a complement to the reference documentation provided in godoc.

guide/.gitignore

@@ -0,0 +1,2 @@
+env/*
+_build/*

guide/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = HCL
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

guide/conf.py

@@ -0,0 +1,164 @@
+import subprocess
+import os.path
+
+# -- Project information -----------------------------------------------------
+
+project = u'HCL'
+copyright = u'2018, HashiCorp'
+author = u'HashiCorp'
+
+git_version = subprocess.check_output(['git', 'describe', '--always']).strip()
+
+# The short X.Y version
+version = unicode(git_version)
+# The full version, including alpha/beta/rc tags
+release = unicode(git_version)
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.todo',
+    'sphinx.ext.githubpages',
+    'sphinxcontrib.golangdomain',
+    'sphinx.ext.autodoc',
+    'autoapi.extension',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', 'env']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'HCLdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'HCL.tex', u'HCL Documentation',
+     u'HashiCorp', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'hcl', u'HCL Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'HCL', u'HCL Documentation',
+     author, 'HCL', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+autoapi_type = 'go'
+autoapi_dirs = [
+    os.path.join(os.path.dirname(__file__), '..', x)
+    for x in ['gohcl', 'hcl', 'hcldec', 'hcled', 'hclparse', 'hcltest', 'hclwrite']
+]
+autoapi_root = 'api'
+autoapi_add_toctree_entry = False
+autoapi_keep_files = True
+autoapi_generate_api_docs = False

guide/go.rst

@@ -0,0 +1,27 @@
+Using HCL in a Go application
+=============================
+
+HCL is itself written in Go_ and currently it is primarily intended for use as
+a library within other Go programs.
+
+This section describes a number of different ways HCL can be used to define
+and process a configuration language within a Go program. For simple situations,
+HCL can decode directly into Go ``struct`` values in a similar way as encoding
+packages such as ``encoding/json`` and ``encoding/xml``.
+
+The HCL Go API also offers some alternative approaches however, for processing
+languages that may be more complex or that include portions whose expected
+structure cannot be determined until runtime.
+
+The following sections give an overview of different ways HCL can be used in
+a Go program.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Sub-sections:
+
+   go_parsing
+   go_diagnostics
+   go_decoding_gohcl
+
+.. _Go: https://golang.org/

guide/go_decoding_gohcl.rst

@@ -0,0 +1,126 @@
+.. go:package:: gohcl
+
+Decoding Into Native Go Values
+==============================
+
+The most straightforward way to access the content of an HCL file is to
+decode into native Go values using ``reflect``, similar to the technique used
+by packages like ``encoding/json`` and ``encoding/xml``.
+
+Package ``gohcl`` provides functions for this sort of decoding. Function
+``DecodeBody`` attempts to extract values from an HCL *body* and write them
+into a Go value given as a pointer:
+
+.. code-block:: go
+
+   type ServiceConfig struct {
+     Type       string `hcl:"type,label"`
+     Name       string `hcl:"name,label"`
+     ListenAddr string `hcl:"listen_addr"`
+   }
+   type Config struct {
+     IOMode   string          `hcl:"io_mode"`
+     Services []ServiceConfig `hcl:"service,block"`
+   }
+
+   var c Config
+   moreDiags := gohcl.DecodeBody(f.Body, nil, &c)
+   diags = append(diags, moreDiags...)
+
+The above example decodes the *root body* of a file ``f``, presumably loaded
+previously using a parser, into the variable ``c``. The field labels within
+the struct types imply the schema of the expected language, which is a cut-down
+version of the hypothetical language we showed in :ref:`intro`.
+
+The struct field labels consist of two comma-separated values. The first is
+the name of the corresponding argument or block type as it will appear in
+the input file, and the second is the type of element being named. If the
+second value is omitted, it defaults to ``attr``, requesting an attribute.
+
+Nested blocks are represented by a struct or a slice of that struct, and the
+special element type ``label`` within that struct declares that each instance
+of that block type must be followed by one or more block labels. In the above
+example, the ``service`` block type is defined to require two labels, named
+``type`` and ``name``. For label fields in particular, the given name is used
+only to refer to the particular label in error messages when the wrong number
+of labels is used.
+
+By default, all declared attributes and blocks are considered to be required.
+An optional value is indicated by making its field have a pointer type, in
+which case ``nil`` is written to indicate the absense of the argument.
+
+The sections below discuss some additional decoding use-cases. For full details
+on the `gohcl` package, see
+`the godoc reference <https://godoc.org/github.com/hashicorp/hcl2/gohcl>`_.
+
+Variables and Functions
+-----------------------
+
+By default, arguments given in the configuration may use only literal values
+and the built in expression language operators, such as arithmetic.
+
+The second argument to ``gohcl.DecodeBody``, shown as ``nil`` in the previous
+example, allows the calling application to additionally offer variables and
+functions for use in expressions. Its value is a pointer to an
+``hcl.EvalContext``, which will be covered in more detail in the later section
+:ref:`expression-eval`. For now, a simple example of making the id of the
+current process available as a single variable called ``pid``:
+
+.. code-block:: go
+
+   type Context struct {
+       Pid string
+   }
+   ctx := gohcl.EvalContext(&Context{
+       Pid: os.Getpid()
+   })
+   var c Config
+   moreDiags := gohcl.DecodeBody(f.Body, ctx, &c)
+   diags = append(diags, moreDiags...)
+
+``gohcl.EvalContext`` constructs an expression evaluation context from a Go
+struct value, making the fields available as variables and the methods
+available as functions, after transforming the field and method names such
+that each word (starting with an uppercase letter) is all lowercase and
+separated by underscores.
+
+.. code-block:: hcl
+
+   name = "example-program (${pid})"
+
+Partial Decoding
+----------------
+
+In the examples so far, we've extracted the content from the entire input file
+in a single call to ``DecodeBody``. This is sufficient for many simple
+situations, but sometimes different parts of the file must be evaluated
+separately. For example:
+
+* If different parts of the file must be evaluated with different variables
+  or functions available.
+
+* If the result of evaluating one part of the file is used to set variables
+  or functions in another part of the file.
+
+There are several ways to perform partial decoding with ``gohcl``, all of
+which involve decoding into HCL's own types, such as ``hcl.Body``.
+
+The most general approach is to declare an additional struct field of type
+``hcl.Body``, with the special field tag type ``remain``:
+
+.. code-block:: go
+
+   type ServiceConfig struct {
+     Type       string   `hcl:"type,label"`
+     Name       string   `hcl:"name,label"`
+     ListenAddr string   `hcl:"listen_addr"`
+     Remain     hcl.Body `hcl:",remain"`
+   }
+
+When a ``remain`` field is present, any element of the input body that is
+not matched is retained in a body saved into that field, which can then be
+decoded in a later call, potentially with a different evaluation context.
+
+Another option is to decode an attribute into a value of type `hcl.Expression`,
+which can then be evaluated separately as described in
+:ref:`expression-eval`.

guide/go_diagnostics.rst

@@ -0,0 +1,97 @@
+.. _go-diagnostics:
+
+Diagnostic Messages
+===================
+
+An important concern for any machine language intended for human authoring is
+to produce good error messages when the input is somehow invalid, or has
+other problems.
+
+HCL uses *diagnostics* to describe problems in an end-user-oriented manner,
+such that the calling application can render helpful error or warning messages.
+The word "diagnostic" is a general term that covers both errors and warnings,
+where errors are problems that prevent complete processing while warnings are
+possible concerns that do not block processing.
+
+HCL deviates from usual Go API practice by returning its own ``hcl.Diagnostics``
+type, instead of Go's own ``error`` type. This allows functions to return
+warnings without accompanying errors while not violating the usual expectation
+that the absense of errors is indicated by a nil ``error``.
+
+In order to easily accumulate and return multiple diagnostics at once, the
+usual pattern for functions returning diagnostics is to gather them in a
+local variable and then return it at the end of the function, or possibly
+earlier if the function cannot continue due to the problems.
+
+.. code-block:: go
+
+  func returningDiagnosticsExample() hcl.Diagnostics {
+      var diags hcl.Diagnostics
+
+      // ...
+
+      // Call a function that may itself produce diagnostics.
+      f, moreDiags := parser.LoadHCLFile("example.conf")
+      // always append, in case warnings are present
+      diags = append(diags, moreDiags...)
+      if diags.HasErrors() {
+        // If we can't safely continue in the presence of errors here, we
+        // can optionally return early.
+        return diags
+      }
+
+      // ...
+
+      return diags
+  }
+
+A common variant of the above pattern is calling another diagnostics-generating
+function in a loop, using ``continue`` to begin the next iteration when errors
+are detected, but still completing all iterations and returning the union of
+all of the problems encountered along the way.
+
+In :ref:`go-parsing`, we saw that the parser can generate diagnostics which
+are related to syntax problems within the loaded file. Further steps to decode
+content from the loaded file can also generate diagnostics related to *semantic*
+problems within the file, such as invalid expressions or type mismatches, and
+so a program using HCL will generally need to accumulate diagnostics across
+these various steps and then render them in the application UI somehow.
+
+Rendering Diagnostics in the UI
+-------------------------------
+
+The best way to render diagnostics to an end-user will depend a lot on the
+type of application: they might be printed into a terminal, written into a
+log for later review, or even shown in a GUI.
+
+HCL leaves the responsibility for rendering diagnostics to the calling
+application, but since rendering to a terminal is a common case for command-line
+tools, the `hcl` package contains a default implementation of this in the
+form of a "diagnostic text writer":
+
+.. code-block:: go
+
+   wr := hcl.NewDiagnosticTextWriter(
+       os.Stdout,      // writer to send messages to
+       parser.Files(), // the parser's file cache, for source snippets
+       78,             // wrapping width
+       true,           // generate colored/highlighted output
+   )
+   wr.WriteDiagnostics(diags)
+
+This default implementation of diagnostic rendering includes relevant lines
+of source code for context, like this:
+
+::
+
+  Error: Unsupported block type
+
+    on example.tf line 4, in resource "aws_instance" "example":
+     2: provisionr "local-exec" {
+
+  Blocks of type "provisionr" are not expected here. Did you mean "provisioner"?
+
+If the "color" flag is enabled, the severity will be additionally indicated by
+a text color and the relevant portion of the source code snippet will be
+underlined to draw further attention.
+

guide/go_parsing.rst

@@ -0,0 +1,64 @@
+.. _go-parsing:
+
+Parsing HCL Input
+=================
+
+The first step in processing HCL input provided by a user is to parse it.
+Parsing turns the raw bytes from an input file into a higher-level
+representation of the arguments and blocks, ready to be *decoded* into an
+application-specific form.
+
+The main entry point into HCL parsing is :go:pkg:`hclparse`, which provides
+:go:type:`hclparse.Parser`:
+
+.. code-block:: go
+
+  parser := hclparse.NewParser()
+  f, diags := parser.ParseHCLFile("server.conf")
+
+Variable ``f`` is then a pointer to an :go:type:`hcl.File`, which is an
+opaque abstract representation of the file, ready to be decoded.
+
+Variable ``diags`` describes any errors or warnings that were encountered
+during processing; HCL conventionally uses this in place of the usual ``error``
+return value in Go, to allow returning a mixture of multiple errors and
+warnings together with enough information to present good error messages to the
+user. We'll cover this in more detail in the next section,
+:ref:`go-diagnostics`.
+
+.. go:package:: hclparse
+
+Package ``hclparse``
+--------------------
+
+.. go:type:: Parser
+
+  .. go:function:: func NewParser() *Parser
+
+      Constructs a new parser object. Each parser contains a cache of files
+      that have already been read, so repeated calls to load the same file
+      will return the same object.
+
+  .. go:function:: func (*Parser) ParseHCL(src []byte, filename string) (*hcl.File, hcl.Diagnostics)
+
+     Parse the given source code as HCL native syntax, saving the result into
+     the parser's file cache under the given filename.
+
+  .. go:function:: func (*Parser) ParseHCLFile(filename string) (*hcl.File, hcl.Diagnostics)
+
+     Parse the contents of the given file as HCL native syntax. This is a
+     convenience wrapper around ParseHCL that first reads the file into memory.
+
+  .. go:function:: func (*Parser) ParseJSON(src []byte, filename string) (*hcl.File, hcl.Diagnostics)
+
+     Parse the given source code as JSON syntax, saving the result into
+     the parser's file cache under the given filename.
+
+  .. go:function:: func (*Parser) ParseJSONFile(filename string) (*hcl.File, hcl.Diagnostics)
+
+     Parse the contents of the given file as JSON syntax. This is a
+     convenience wrapper around ParseJSON that first reads the file into memory.
+
+The above list just highlights the main functions in this package.
+For full documentation, see
+`the hclparse godoc <https://godoc.org/github.com/hashicorp/hcl2/hclparse>`_.

guide/index.rst

@@ -0,0 +1,34 @@
+HCL Configuration Language
+==========================
+
+HCL is a toolkit for creating structured configuration languages that are both
+human- and machine-friendly, for use with command-line tools, servers, etc.
+
+HCL has both a native syntax, intended to be pleasant to read and write for
+humans, and a JSON-based variant that is easier for machines to generate and
+parse. The native syntax is inspired by libucl_, `nginx configuration`_, and
+others.
+
+It includes an expression syntax that allows basic inline computation and, with
+support from the calling application, use of variables and functions for more
+dynamic configuration languages.
+
+HCL provides a set of constructs that can be used by a calling application to
+construct a configuration language. The application defines which argument
+names and nested block types are expected, and HCL parses the configuration
+file, verifies that it conforms to the expected structure, and returns
+high-level objects that the application can use for further processing.
+
+At present, HCL is primarily intended for use in applications written in Go_,
+via its library API.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contents:
+
+   intro
+   go
+
+.. _libucl: https://github.com/vstakhov/libucl
+.. _`nginx configuration`: http://nginx.org/en/docs/beginners_guide.html#conf_structure
+.. _Go: https://golang.org/

guide/intro.rst

@@ -0,0 +1,108 @@
+.. _intro:
+
+Introduction to HCL
+===================
+
+HCL-based configuration is built from two main constructs: arguments and
+blocks. The following is an example of a configuration language for a
+hypothetical application:
+
+.. code-block:: hcl
+
+  io_mode = "async"
+
+  service "http" "web_proxy" {
+    listen_addr = "127.0.0.1:8080"
+
+    process "main" {
+      command = ["/usr/local/bin/awesome-app", "server"]
+    }
+
+    process "mgmt" {
+      command = ["/usr/local/bin/awesome-app", "mgmt"]
+    }
+  }
+
+In the above example, ``io_mode`` is a top-level argument, while ``service``
+introduces a block. Within the body of a block, further arguments and nested
+blocks are allowed. A block type may also expect a number of *labels*, which
+are the quoted names following the ``service`` keyword in the above example.
+
+The specific keywords ``io_mode``, ``service``, ``process``, etc here are
+application-defined. HCL provides the general block structure syntax, and
+can validate and decode configuration based on the application's provided
+schema.
+
+HCL is a structured configuration language rather than a data structure
+serialization language. This means that unlike languages such as JSON, YAML,
+or TOML, HCL is always decoded using an application-defined schema.
+
+However, HCL does have a JSON-based alternative syntax, which allows the same
+structure above to be generated using a standard JSON serializer when users
+wish to generate configuration programmatically rather than hand-write it:
+
+.. code-block:: json
+
+  {
+    "io_mode": "async",
+    "service": {
+      "http": {
+        "web_proxy": {
+          "listen_addr": "127.0.0.1:8080",
+          "process": {
+            "main": {
+              "command": ["/usr/local/bin/awesome-app", "server"]
+            },
+            "mgmt": {
+              "command": ["/usr/local/bin/awesome-app", "mgmt"]
+            },
+          }
+        }
+      }
+    }
+  }
+
+The calling application can choose which syntaxes to support. JSON syntax may
+not be important or desirable for certain applications, but it is available for
+applications that need it. The schema provided by the calling application
+allows JSON input to be properly decoded even though JSON syntax is ambiguous
+in various ways, such as whether a JSON object is representing a nested block
+or an object expression.
+
+The collection of arguments and blocks at a particular nesting level is called
+a *body*. A file always has a root body containing the top-level elements,
+and each block also has its own body representing the elements within it.
+
+The term "attribute" can also be used to refer to what we've called an
+"argument" so far. The term "attribute" is also used for the fields of an
+object value in argument expressions, and so "argument" is used to refer
+specifically to the type of attribute that appears directly within a body.
+
+The above examples show the general "texture" of HCL-based configuration. The
+full details of the syntax are covered in the language specifications.
+
+.. todo:: Once the language specification documents have settled into a
+   final location, link them from above.
+
+Argument Expressions
+--------------------
+
+The value of an argument can be a literal value shown above, or it may be an
+expression to allow arithmetic, deriving one value from another, etc.
+
+.. code-block:: hcl
+
+  listen_addr = env.LISTEN_ADDR
+
+Built-in arithmetic and comparison operators are automatically available in all
+HCL-based configuration languages. A calling application may optionally
+provide variables that users can reference, like ``env`` in the above example,
+and custom functions to transform values in application-specific ways.
+
+Full details of the expression syntax are in the HCL native syntax
+specification. Since JSON does not have an expression syntax, JSON-based
+configuration files use the native syntax expression language embedded inside
+JSON strings.
+
+.. todo:: Once the language specification documents have settled into a
+   final location, link to the native syntax specification from above.

guide/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=HCL
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

guide/requirements.txt

@@ -0,0 +1,3 @@
+sphinx
+sphinxcontrib-golangdomain
+sphinx-autoapi