Cleanup
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
## Introduction
|
||||
|
||||
We wish to print a list of all the files in a directory, ignoring files beginning with certain characters, sorted alphabetically, with the directories last.
|
||||
|
||||
## Finding the files
|
||||
|
||||
### Code
|
||||
|
||||
Finds all files in the `path` directory recursively, ignoring any files or directories that start with any of the characters in the `@starts_with` module attribute.
|
||||
|
||||
```elixir
|
||||
defmodule Files do
|
||||
@starts_with [".", "_"]
|
||||
|
||||
def find(path \\ "."), do: find(path, list_contents(path))
|
||||
|
||||
defp find(dir, files) do
|
||||
Enum.reduce(files, {dir, []}, fn file, {path, contents} ->
|
||||
{path, path |> Path.join(file) |> update_contents(file, contents)}
|
||||
end)
|
||||
end
|
||||
|
||||
defp list_contents(path), do: path |> File.ls!() |> ignore()
|
||||
|
||||
defp update_contents(path, file, contents) do
|
||||
cond do
|
||||
File.regular?(path) -> [file | contents]
|
||||
File.dir?(path) -> [find(path) | contents]
|
||||
end
|
||||
end
|
||||
|
||||
defp ignore(filenames) do
|
||||
Enum.reject(filenames, &(String.first(&1) in @starts_with))
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
The data structure holding the results of the file search.
|
||||
|
||||
```
|
||||
iex(1)> directory_tree = Files.find("hello")
|
||||
{"hello",
|
||||
[
|
||||
"README.md",
|
||||
{"hello/test",
|
||||
[
|
||||
{"hello/test/support", ["conn_case.ex"]},
|
||||
"test_helper.exs",
|
||||
{"hello/test/hello_web",
|
||||
[{"hello/test/hello_web/controllers", ["error_json_test.exs"]}]}
|
||||
]},
|
||||
{"hello/lib",
|
||||
[
|
||||
"hello.ex",
|
||||
{"hello/lib/hello", ["application.ex"]},
|
||||
"hello_web.ex",
|
||||
{"hello/lib/hello_web",
|
||||
[
|
||||
{"hello/lib/hello_web/controllers", ["error_json.ex"]},
|
||||
"telemetry.ex",
|
||||
"router.ex",
|
||||
"endpoint.ex"
|
||||
]}
|
||||
]},
|
||||
{"hello/priv", [{"hello/priv/static", ["robots.txt", "favicon.ico"]}]},
|
||||
{"hello/config",
|
||||
["config.exs", "dev.exs", "test.exs", "prod.exs", "runtime.exs"]},
|
||||
"mix.exs"
|
||||
]}
|
||||
```
|
||||
|
||||
## Sorting and printing
|
||||
|
||||
### Code
|
||||
|
||||
Sort alphabetically, with directories last. Downcase before comparing strings.
|
||||
|
||||
```elixir
|
||||
defmodule Paths do
|
||||
def puts(dir_tree), do: dir_tree |> print() |> Enum.join("\n") |> IO.puts()
|
||||
|
||||
defp print({path, contents}), do: path |> list(contents) |> List.flatten()
|
||||
|
||||
defp list(path, contents) do
|
||||
contents |> Enum.sort(&alpha_asc_dir_last/2) |> Enum.map(&make_path(&1, path))
|
||||
end
|
||||
|
||||
defp alpha_asc_dir_last({a, _}, {b, _}), do: fmt(a) < fmt(b)
|
||||
defp alpha_asc_dir_last({_, _}, _), do: false
|
||||
defp alpha_asc_dir_last(_, {_, _}), do: true
|
||||
defp alpha_asc_dir_last(a, b), do: fmt(a) < fmt(b)
|
||||
|
||||
defp make_path(filename, path) when is_binary(filename), do: Path.join(path, filename)
|
||||
defp make_path({path, contents}, _), do: list(path, contents)
|
||||
|
||||
defp fmt(f), do: String.downcase(f)
|
||||
end
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
Print all the files sorted.
|
||||
|
||||
```
|
||||
iex(2)> Paths.puts(directory_tree)
|
||||
hello/mix.exs
|
||||
hello/README.md
|
||||
hello/config/config.exs
|
||||
hello/config/dev.exs
|
||||
hello/config/prod.exs
|
||||
hello/config/runtime.exs
|
||||
hello/config/test.exs
|
||||
hello/lib/hello.ex
|
||||
hello/lib/hello_web.ex
|
||||
hello/lib/hello/application.ex
|
||||
hello/lib/hello_web/endpoint.ex
|
||||
hello/lib/hello_web/router.ex
|
||||
hello/lib/hello_web/telemetry.ex
|
||||
hello/lib/hello_web/controllers/error_json.ex
|
||||
hello/priv/static/favicon.ico
|
||||
hello/priv/static/robots.txt
|
||||
hello/test/test_helper.exs
|
||||
hello/test/hello_web/controllers/error_json_test.exs
|
||||
hello/test/support/conn_case.ex
|
||||
:ok
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
|
||||
Trying to do this without recursion made it difficult to sort directories first.
|
||||
44
posts/2023-09-15-open-an-iex-shell-from-an-elixir-script.md
Normal file
44
posts/2023-09-15-open-an-iex-shell-from-an-elixir-script.md
Normal file
@@ -0,0 +1,44 @@
|
||||
---
|
||||
title: "Open An IEx Shell From An Elixir Script"
|
||||
blurb: "We can run an Elixir script with either the `elixir` or the
|
||||
`iex` command. Both will execute the code, but the second command
|
||||
opens an interactive IEx shell afterward. What if, we won't know until runtime
|
||||
whether we want a shell or not? How can we start an IEx session even when we
|
||||
use `elixir`, instead of `iex`, to run our script?"
|
||||
...
|
||||
|
||||
I recently had occasion to want to start an IEx session from an Elixir script. Here's how I was able to do it.
|
||||
|
||||
## Method 1
|
||||
|
||||
Here's a quick test script:
|
||||
|
||||
<p class="code-filename-label">`run.exs`</p>
|
||||
|
||||
```elixir
|
||||
:shell.start_interactive({IEx, :start, []})
|
||||
|
||||
System.no_halt(true)
|
||||
```
|
||||
|
||||
`System.no_halt(true)` is needed so that the interactive session doesn't die when the process running the script ends. You could also use the `elixir` command with the `--no-halt` option instead to achieve the same effect.
|
||||
|
||||
When we run this script with just `elixir` (instead of `iex`), we should still see an interactive IEx session start.
|
||||
|
||||
```bash
|
||||
$ elixir run.exs
|
||||
Erlang/OTP 26 [erts-14.1] [source] [64-bit] [smp:12:12] [ds:12:12:10] [async-threads:1] [jit:ns]
|
||||
|
||||
Interactive Elixir (1.15.6) - press Ctrl+C to exit (type h() ENTER for help)
|
||||
iex(1)>
|
||||
```
|
||||
|
||||
## Method 2
|
||||
|
||||
While researching, I found another way that works, but it's even more obscure than the above method.
|
||||
|
||||
```elixir
|
||||
:user_drv.start_shell(%{initial_shell: {IEx, :start, []}})
|
||||
```
|
||||
|
||||
I can't find documentation for `user_drv`, I had to browse the [code](https://github.com/erlang/otp/blob/master/lib/kernel/src/user_drv.erl) directly.
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
title: "Start Erlang's Dialyzer With GUI From A Docker Container"
|
||||
blurb: "Everything in OTP is command-line driven, so using containers during development has been without issue. But, Dialyzer, Erlang's static analysis tool, actually has a Graphical User Interface. How can we still use Dialyzer and its GUI even though Elixir is running inside a container?"
|
||||
...
|
||||
|
||||
I use Docker mostly when working on software projects and I figured out how to get Erlang's Dialyzer GUI working in a Docker container.
|
||||
|
||||
1. Start a container with X11 forwarding:
|
||||
|
||||
```
|
||||
$ docker run -it --rm --net=host -e "DISPLAY" -e "NO_AT_BRIDGE=1" \
|
||||
-v "$XAUTHORITY:/root/.Xauthority:rw" elixir bash
|
||||
```
|
||||
|
||||
2. Build ["the recommended minimal PLT for Erlang/OTP"](https://www.erlang.org/doc/apps/dialyzer/dialyzer_chapter#the-persistent-lookup-table):
|
||||
|
||||
```
|
||||
# dialyzer --build_plt --apps erts kernel stdlib mnesia
|
||||
Creating PLT /root/.cache/erlang/.dialyzer_plt ...
|
||||
...
|
||||
done (warnings were emitted)
|
||||
```
|
||||
|
||||
3. Start the Dialyzer GUI:
|
||||
|
||||
a. From the command line:
|
||||
|
||||
# dialyzer --gui
|
||||
|
||||
b. From inside an IEx session:
|
||||
|
||||
1. Start the Dialyzer application:
|
||||
|
||||
```elixir
|
||||
iex(1)> Application.load(:dialyzer)
|
||||
:ok
|
||||
```
|
||||
|
||||
2. Start the Dialyzer GUI:
|
||||
|
||||
```elixir
|
||||
iex(2)> :dialyzer.gui()
|
||||
```
|
||||
|
||||
You should now see the Dialyzer GUI.
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
blurb: "One thing we learned from a week of trying to make isometric vector drawings."
|
||||
...
|
||||
|
||||
## Objective
|
||||
|
||||
Construct an isometric vector image of an object from top, front, and side view projections (like [this](https://workforce.libretexts.org/Bookshelves/Drafting_and_Design_Technology/Introduction_to_Drafting_and_AutoCAD_3D_(Baumback)/02%3A_Part_2/2.01%3A_Module_7-_Visualizing_Multiview_Drawings), but using Inkscape instead of AutoCAD).
|
||||
|
||||
This is an explanation of one of the issues encountered and how it was dealt with.
|
||||
|
||||
## A Short Primer On Graphical Projection
|
||||
|
||||
Drawing a 3D object as it looks head-on from the top, the front, and the side views is called [multiview orthographic projection](https://en.wikipedia.org/wiki/Multiview_orthographic_projection){target=_local}. Figure 1, below, is [an example from 1912](https://commons.wikimedia.org/wiki/File:3D_projection_views.png#/media/File:3D_projection_views.png){target=_local} that illustrates the concept. Each drawing is a view of the house from a different axis in three-dimensional space. Note the various other names, by which these views are known, indicated on the labeling.
|
||||
|
||||
{width=300px}
|
||||
|
||||
Another type of graphical projection, that you may have seen before, is called [isometric](https://en.wikipedia.org/wiki/Isometric_projection). Many 2D video games use isometric projection to simulate a 3D environment. One of the characteristics of isometric projection is, the angles between all three axes are equal. This results in the outline of a perfect cube forming a hexagon. Another is, it has no vanishing point, so there is no concern for things like draw-distance as there is in 3D games.
|
||||
|
||||
|
||||
|
||||
## A Naive Solution
|
||||
|
||||
We start with the multiview projections of the 3D object we want to turn into an isometric drawing:
|
||||
|
||||
|
||||
|
||||
The task seems pretty straight-forward. First, we skew the front and side views. We want our truck to face to the right, so we need the top edge of the front view to match the angle of the blue axis from figure 2. To reach this angle, the skew must be 30°. The angle of the top of the side view must, therefore, match the red axis from figure 2. To achieve this, it has to be skewed 30° in the other direction (-30°, technically).
|
||||
|
||||
|
||||
|
||||
Then, we transform the top view to fit our modified front and side views.
|
||||
|
||||
|
||||
|
||||
All the angles for our box truck's skewed views look correct, but when we try to align them all together, we discover a problem. While the front and top views line up fine, when we try to align the top to the side view, we find that they do not fit together because they are different sizes.
|
||||
|
||||
|
||||
|
||||
## The Problem
|
||||
|
||||
The pieces don't fit together because when we skew a 2-dimensional rectangle we are actually stretching one of the axes. The top and bottom edges of the skewed side-face are longer than they were before the transformation.
|
||||
|
||||
|
||||
|
||||
The skewed front and top views of our box truck also become stretched on one axis but because they both stretch the same axis, the pieces still fit together.
|
||||
|
||||
## The Quick Fix
|
||||
|
||||
We can fix the box truck's elongated side view by resizing it before we skew it. We need to shrink it along the axis that is going to be stretched by the skew. But how do we know what value to scale by? Here's how to find it:
|
||||
|
||||
1. 
|
||||
Draw a line the same length as the truck side view.
|
||||
|
||||
2. 
|
||||
Duplicate the line and rotate it by 30°.
|
||||
|
||||
3. 
|
||||
Join one end of the two lines together.
|
||||
|
||||
4. 
|
||||
Measure the horizontal distance between the other ends of the lines. This is the amount the side must be shrunk by so that it will be the correct size when stretched by skewing.
|
||||
|
||||
5. 
|
||||
Shrink the side view by the distance we measured.
|
||||
|
||||
6. 
|
||||
Now we can skew the side as we did before.
|
||||
|
||||
This time when we try to line up the 3 skewed views of our isometric drawing, they should fit together.
|
||||
|
||||
|
||||
|
||||
However, all the distortion in our render has not been eliminated. While this solution might look good, it is not true proportionally to the top and front views. They are still distorted from the skew along the axis perpendicular to the side view.
|
||||
|
||||
## The Full Fix
|
||||
|
||||
So, if the top and front views are still distorted, we will have to size them down and then re-skew them.
|
||||
|
||||
Comparing the edges where the front view and the top view meet, we see that they are actually the same length. To find the distance the views need to be shrunk by, we will only need to make this calculation once.
|
||||
|
||||
|
||||
|
||||
Now we can shrink the front view and then skew it as we did before.
|
||||
|
||||
|
||||
|
||||
Likewise, we shrink the top view's width (in this case, this is the height of the diagram), skew it, and rotate it to fit.
|
||||
|
||||
|
||||
|
||||
When we put all the pieces together, we can see that this version of our isometric model is less wide than before. This is because its width is no longer being stretched out of proportion.
|
||||
|
||||
|
||||
|
||||
## Practical Applications
|
||||
|
||||
Is this useful? Multiview projections of vehicles, buildings, etc. are easy to find online. If you want to build proportionally-accurate isometric drawings then this information will help. For example, here is an isometric taxi cab made with this technique and some found [diagrams](https://drawingdatabase.com/checker-marathon-taxi/).
|
||||
|
||||
|
||||
|
||||
## Conclusion
|
||||
|
||||
Making isometric drawings from blueprints of real objects is more fun and enjoyable when they come out looking like the real thing. Using the technique outlined here should eliminate distortion introduced from skewing multiview projections and retain their true proportions. Hopefully, it will improve your work as it has done mine.
|
||||
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
---
|
||||
title: "Deploy Elixir-Generated HTML With Docker On DigitalOcean"
|
||||
blurb: "This is a simple proof of concept where we create a boilerplate HTML file with Elixir, containerize our build process with Docker, and deploy our markup live with DigitalOcean's hosting service."
|
||||
...
|
||||
|
||||
## Introduction
|
||||
|
||||
DigitalOcean has this [App Platform](https://www.digitalocean.com/products/app-platform) service that can host a static website, as well as build it from a Docker image, if provided a `Dockerfile`. We thought a static website built by an Elixir app could be an instructive project. To explore if the idea is viable, we wrote a small Elixir application that generates a simple `index.html` file and deployed it live on DigitalOcean's service.
|
||||
|
||||
## Requirements
|
||||
|
||||
* Docker
|
||||
* GitHub account
|
||||
* DigitalOcean account
|
||||
|
||||
This is not an endorsement of these vendors. We don't have special affinity for any of them. We just like containers, git, and PaaS. This post is specific to DigitalOcean, though, because that's where we deploy our site.
|
||||
|
||||
## Procedure
|
||||
|
||||
The instructions are divided into four parts.
|
||||
|
||||
1. [Create the mix project](#create-the-mix-project)
|
||||
2. [Create the build task](#create-the-build-task)
|
||||
3. [Add the `Dockerfile`](#add-the-dockerfile)
|
||||
4. [Deploy live](#deploy-live)
|
||||
|
||||
::: warning
|
||||
We're going to do this whole thing with containers. That means Docker is all we need to have installed on our machines. Be warned, though, that also means we'll be running Elixir through the `docker` command.
|
||||
:::
|
||||
|
||||
### 1. Create the mix project
|
||||
|
||||
We use the `mix new` command to create our Elixir project. If we were using Elixir installed natively on our computer, it would just be the last part, `mix new static_site`. Since we are using Docker, the command looks like this:
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt -u $(id -u):$(id -u) elixir mix new static_site
|
||||
```
|
||||
|
||||
That might look a bit overwhelming, so let's explain each part of the command.
|
||||
|
||||
* `docker run elixir`
|
||||
|
||||
`docker run` creates and starts a new container from an image. `elixir` uses the latest [official Docker elixir image](https://hub.docker.com/_/elixir).
|
||||
|
||||
* `--rm`
|
||||
|
||||
Since `docker run` creates a new container just to run this single command, this option will delete the container once it has finished. Using `--rm` keeps us from creating a bunch of containers that have to be cleaned up later.
|
||||
|
||||
* `-v $PWD:/opt`
|
||||
|
||||
We want the files Mix generates to be accessible to us so we can edit and keep them in version control. We need to bring them out of the container somehow. We do this by mount binding a volume. This option binds the result of `$PWD`, which is the current directory on our filesystem, to the `/opt` directory on the container filesystem. Any changes made in the container to the `/opt` directory will be reflected on our filesystem.
|
||||
|
||||
* `-w /opt`
|
||||
|
||||
This option sets the directory that the command will run in. Since we mounted our project files in the container's `/opt` directory, we want to set it as the working directory.
|
||||
|
||||
* `-u $(id -u):$(id -u)`
|
||||
|
||||
This option sets the container user and group to match our operating system's current user. If we don't do this, the files generated will all belong to `root` and be uneditable to us without using `sudo`.
|
||||
|
||||
* `mix new static_site`
|
||||
|
||||
The command we want to run in the container.
|
||||
|
||||
After the command finishes, we have a new directory on our filesystem called `static_site`. Let's change into that directory and make sure we can run the tests. We don't care if the files `mix` creates in `_build` are owned by `root`, so we don't bother setting the user and group with the `-u` option when we run the command this time.
|
||||
|
||||
```
|
||||
$ cd static_site
|
||||
$ docker run --rm -w /opt -v $PWD:/opt elixir mix test
|
||||
```
|
||||
|
||||
We should see a successful test result.
|
||||
|
||||
With our Mix project files generated, we move on to implementing creating our static HTML file.
|
||||
|
||||
### 2. Create the build task
|
||||
|
||||
Because our output will only contain static markup, our Elixir application will not be a long running process in production. It will only run once, during the build phase of deployment. A one-off job is the perfect role for a Mix task. The [`Task` module documentation](https://hexdocs.pm/mix/Mix.Task.html) shows an example file to start with and even tells us where to put it (`lib/mix/tasks`).
|
||||
|
||||
We name the task simply `build` and create a file called `build.ex`. It uses Elixir's `File` module to first create a directory called `/public`. Then, it writes a minimal `index.html` file at that location.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/mix/tasks/build.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Mix.Tasks.Build do
|
||||
@moduledoc "Creates a /public directory and places an HTML index file there"
|
||||
@shortdoc "Builds static HTML file"
|
||||
|
||||
use Mix.Task
|
||||
|
||||
@output_directory "/public"
|
||||
|
||||
@markup """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head></head>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
@impl Mix.Task
|
||||
def run(_args) do
|
||||
Mix.shell().info("running build task")
|
||||
|
||||
File.mkdir_p!(@output_directory)
|
||||
@output_directory |> Path.join("index.html") |> File.write!(@markup)
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
The easiest way to test our task is to run `mix build` and then inspect the contents of `/public/index.html` with the `cat` command, but `docker run` only accepts a single command. We can combine both into one with `bash -c "mix build && cat /public/index.html"`.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt elixir bash -c "mix build && cat /public/index.html"
|
||||
```
|
||||
|
||||
If all went well, our output should be:
|
||||
|
||||
```
|
||||
running build task
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head></head>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
With creating the Mix task complete, it is time to add a `Dockerfile` to our project.
|
||||
|
||||
### 3. Add the `Dockerfile`
|
||||
|
||||
We could commit HTML files to our repo directly and deploy them that way. It would not require Docker at all. But if we want to use Elixir to generate hypertext markup programatically, we will have to add a `Dockerfile` for building our project in production.
|
||||
|
||||
#### Dependencies
|
||||
|
||||
In the last section, we built the HTML file by calling our Mix task with the command, `mix build`. Let's make sure our build handles dependencies by adding one to the project. We add a `plug` dependency in the `mix.exs` file and try building again.
|
||||
|
||||
<p class="code-filename-label">`mix.exs`</p>
|
||||
|
||||
```elixir
|
||||
defmodule StaticSite.MixProject do
|
||||
use Mix.Project
|
||||
...
|
||||
|
||||
defp deps do
|
||||
[
|
||||
{:plug, ">= 0.0.0"}
|
||||
]
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
We will also have to add a call to `mix deps.get` in our command. Again, we combine the two commands into one with `&&`:
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt elixir bash -c "mix deps.get && mix build"
|
||||
* creating /root/.mix/archives/hex-2.0.6
|
||||
Resolving Hex dependencies...
|
||||
...
|
||||
* Getting plug (Hex package)
|
||||
...
|
||||
Compiling 2 files (.ex)
|
||||
Generated static_site app
|
||||
running build task
|
||||
```
|
||||
|
||||
We should see hex installing, dependencies being fetched, the project compiled, and our mix task being ran.
|
||||
|
||||
#### Environments
|
||||
|
||||
There is a problem with this method, however. What if the dependencies are only needed during development? We don't want `dev` environment dependencies being included when we deploy to production. To test this, change `{:plug, ">= 0.0.0"}` to `{:plug, ">= 0.0.0", only: :dev}` in `mix.exs`.
|
||||
|
||||
##### Development (`dev`)
|
||||
|
||||
We will re-run our command, but we will add a call to `mix deps` to see what dependencies our project builds.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt elixir bash -c "mix deps.get && mix build && mix deps"
|
||||
...
|
||||
Compiling 2 files (.ex)
|
||||
Generated static_site app
|
||||
running build task
|
||||
* mime 2.0.5 (Hex package) (mix)
|
||||
locked at 2.0.5 (mime) da0d64a3
|
||||
ok
|
||||
* plug 1.15.2 (Hex package) (mix)
|
||||
locked at 1.15.2 (plug) 02731fa0
|
||||
ok
|
||||
* plug_crypto 2.0.0 (Hex package) (mix)
|
||||
locked at 2.0.0 (plug_crypto) 53695bae
|
||||
ok
|
||||
* telemetry 1.2.1 (Hex package) (rebar3)
|
||||
locked at 1.2.1 (telemetry) dad9ce9d
|
||||
ok
|
||||
```
|
||||
|
||||
Our `dev` environment has the dependency we added.
|
||||
|
||||
##### Production (`prod`)
|
||||
|
||||
Now, we will run the command again after we set the environment variable `MIX_ENV` to `prod`. We do this with the `-e` option:
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt -e MIX_ENV=prod elixir bash -c "mix deps.get && mix build && mix deps"
|
||||
* creating /root/.mix/archives/hex-2.0.6
|
||||
Resolving Hex dependencies...
|
||||
...
|
||||
* Getting plug (Hex package)
|
||||
* Getting mime (Hex package)
|
||||
* Getting plug_crypto (Hex package)
|
||||
* Getting telemetry (Hex package)
|
||||
...
|
||||
Compiling 2 files (.ex)
|
||||
Generated static_site app
|
||||
running build task
|
||||
```
|
||||
|
||||
`mix deps` doesn't list any dependencies in the `prod` environment, which is what we want. However, we don't want it *getting* any dependencies that aren't used, either.
|
||||
|
||||
##### Using the `--only` option
|
||||
|
||||
It turns out that `mix deps.get` [has an `--only` option](https://hexdocs.pm/mix/Mix.Tasks.Deps.Get.html) that can be used to fetch dependencies only for a specific environment. We try our command again with that option.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt -e MIX_ENV=prod elixir bash -c "mix deps.get --only prod && mix build && mix deps"
|
||||
* creating /root/.mix/archives/hex-2.0.6
|
||||
All dependencies are up to date
|
||||
Compiling 2 files (.ex)
|
||||
Generated static_site app
|
||||
running build task
|
||||
```
|
||||
|
||||
We don't see any dependencies being fetched, so that works as we want it to. We can set `MIX_ENV` to `prod` in production and use `mix deps.get --only $MIX_ENV` in our `Dockerfile` to fetch dependencies only belonging to that environment.
|
||||
|
||||
#### Writing the file
|
||||
|
||||
`Dockerfiles` have 2 different ways to use variables, `ENV` and `ARG`. `ENV` variables will be available in the running container, while `ARG` variables are only available during the image build process. Since we only need the variable to be available during the deployment, we need to use `ARG` in our `Dockerfile`. We can even set a default, so that we don't need to set `MIX_ENV` explicitly when we are developing. Here is our complete `Dockerfile`:
|
||||
|
||||
<p class="code-filename-label">`Dockerfile`</p>
|
||||
|
||||
```dockerfile
|
||||
FROM elixir:slim
|
||||
|
||||
WORKDIR /opt
|
||||
COPY lib ./lib
|
||||
COPY mix.exs ./mix.exs
|
||||
|
||||
ARG MIX_ENV=dev
|
||||
|
||||
RUN mix deps.get --only $MIX_ENV && mix build
|
||||
```
|
||||
|
||||
Here's the explanation:
|
||||
|
||||
* `FROM elixir:slim`
|
||||
|
||||
This is the image we are basing ours on. The `slim` version is slightly smaller, so we chose it to save space.
|
||||
|
||||
* `WORKDIR /opt`
|
||||
|
||||
Just like the `-w` option in our `docker run` command, this sets the working directory to `/opt`.
|
||||
|
||||
* `COPY lib ./lib`
|
||||
`COPY mix.exs ./mix.exs`
|
||||
|
||||
These copy our project files into the current working directory (`/opt`).
|
||||
|
||||
* `ARG MIX_ENV=dev`
|
||||
|
||||
Makes the `MIX_ENV` variable available in the `Dockerfile` if it is set in the environment. If it is not set, this tells Docker to use `dev` as the default value.
|
||||
|
||||
* `RUN mix deps.get --only $MIX_ENV && mix build`
|
||||
|
||||
Fetches depencies for our mix project and runs our build task.
|
||||
|
||||
With that complete, we can now build an image.
|
||||
|
||||
#### Building the image
|
||||
|
||||
##### `dev` environment
|
||||
|
||||
This first build will not have the `MIX_ENV` variable set, so we expect it to default to `dev` and to find the `plug` dependency installed.
|
||||
|
||||
```
|
||||
$ docker build -t hw_dev .
|
||||
```
|
||||
|
||||
Let's see what dependencies our image contains:
|
||||
|
||||
```
|
||||
$ docker run --rm hw_dev mix deps
|
||||
|
||||
* mime 2.0.5 (Hex package) (mix)
|
||||
locked at 2.0.5 (mime) da0d64a3
|
||||
ok
|
||||
* plug 1.15.2 (Hex package) (mix)
|
||||
locked at 1.15.2 (plug) 02731fa0
|
||||
ok
|
||||
* plug_crypto 2.0.0 (Hex package) (mix)
|
||||
locked at 2.0.0 (plug_crypto) 53695bae
|
||||
ok
|
||||
* telemetry 1.2.1 (Hex package) (rebar3)
|
||||
locked at 1.2.1 (telemetry) dad9ce9d
|
||||
ok
|
||||
```
|
||||
|
||||
That looks good. And let's also check that our HTML file was written:
|
||||
|
||||
```
|
||||
$ docker run --rm hw_dev ls /public
|
||||
index.html
|
||||
```
|
||||
|
||||
##### `prod` environment
|
||||
|
||||
Excellent. Now let's build a production image. We pass in the value to set for the `MIX_ENV` argument with the `--build-arg` option:
|
||||
|
||||
```
|
||||
$ docker build -t hw_prod --build-arg MIX_ENV=prod .
|
||||
```
|
||||
|
||||
And now we check as before. We have to set `MIX_ENV` in the container if we want our mix command to run in that environment. We do this with the `-e` option.
|
||||
|
||||
```
|
||||
$ docker run --rm -e MIX_ENV=prod hw_prod mix deps
|
||||
$
|
||||
```
|
||||
|
||||
This shows no dependencies, as expected. And check our index.html:
|
||||
|
||||
```
|
||||
$ docker run --rm hw_prod ls /public
|
||||
index.html
|
||||
```
|
||||
|
||||
It works! Now we can deploy.
|
||||
|
||||
### 4. Deploy live
|
||||
|
||||
1. First, we push our git repo to GitHub. It doesn't have to be a public repo, we will give DigitalOcean access to it in a moment.
|
||||
2. We [log in](https://cloud.digitalocean.com/login) to DigitalOcean, go to our Apps page and click "Create App".
|
||||
3. On the "Create Resource From Source Code" screen, we click on "Edit Your GitHub Permissions".
|
||||
4. We do what GitHub calls "installing" the DigitalOcean app on our GitHub account.
|
||||
5. We add the repo to the "Repository access" list and click "Save". We should be redirected back to the DigitalOcean "Create Resource From Source Code" screen.
|
||||
6. We select "GitHub" as the "Service Provider". Our repo should now appear in the list. We select it and click "Next".
|
||||
7. It detected our Dockerfile and thinks we want to run a web service. We need to edit the resource to tell it it's a static site that just needs Docker to build. We click on the "Edit" button.
|
||||
8. Under "Resource Type", we click "Edit" and select "Static Site". Then we click "Save".
|
||||
9. We edit the "Output Directory" and set it to `/public`, the location of the static files in the container. Then click "< Back".
|
||||
10. Under the "App" section, we should see "Starter Plan" and "Static Site". We click "Next".
|
||||
11. On the "Environment Variables" page, we can set `MIX_ENV`, the variable our Dockerfile will need during the build process, to `prod`. It probably works the same whether set under "Global" or local to the app, but we just set it on the `static-site` app.
|
||||
12. We don't have a reason to change anything on the "Info" screen, so we click "Next". If we wanted to change the resource name, this would be the time to do it.
|
||||
13. At the bottom of the "Review" screen, we click "Create Resources".
|
||||
|
||||
After deployment finishes successfully, we should be able to visit the link that DigitalOcean gives us and we will be greeted by our "hello world" HTML page.
|
||||
|
||||
## Making changes
|
||||
|
||||
If we want to make changes, we can commit our updates and push the code to GitHub. This will trigger a rebuild and deploy the changes, automatically.
|
||||
|
||||
## Conclusion
|
||||
|
||||
Our Elixir-generated HTML file is live and hosted. We have completed our proof of concept. If we wanted to take advantage of DigitalOcean's platform and host an Elixir-generated static website, this is the blueprint we could follow. It's relatively simple if familiar with Docker, and once set up, deploying changes with a `git push` is simply magical. We look forward to using what we have learned in a future project.
|
||||
322
posts/2023-11-15-test-mix-task-file-modify.md
Normal file
322
posts/2023-11-15-test-mix-task-file-modify.md
Normal file
@@ -0,0 +1,322 @@
|
||||
---
|
||||
title: "Temporary Directories For Testing Mix Tasks That Modify Files"
|
||||
blurb: "Writing a test for a simple Mix task gets surprisingly complex. Application environment variables, temporary test directories, and IO capture are all involved."
|
||||
...
|
||||
|
||||
## Intro
|
||||
|
||||
Last time, we added a Mix task to our project that writes an HTML file to a directory `/public` in the container's filesystem. Today, we will write a test for that task.
|
||||
|
||||
Here is the code we want to write a test for.
|
||||
|
||||
```elixir
|
||||
@markup """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head></head>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
@output_directory "/public"
|
||||
|
||||
@impl Mix.Task
|
||||
def run(_args) do
|
||||
Mix.shell().info("running build task")
|
||||
|
||||
File.mkdir_p!(@output_directory)
|
||||
@output_directory |> Path.join("index.html") |> File.write!(@markup)
|
||||
end
|
||||
```
|
||||
|
||||
## First pass
|
||||
|
||||
The task writes some HTML markup to a file, `/public/index.html`, so our test should ensure that after the task is ran, that file exists.
|
||||
|
||||
::: filename-for-code-block
|
||||
`test/mix/tasks/build_test.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Mix.Tasks.BuildTest do
|
||||
use ExUnit.Case
|
||||
|
||||
test "creates output files" do
|
||||
Mix.Tasks.Build.run([])
|
||||
|
||||
assert File.exists?("/public/index.html")
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
Let's try running the test.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt hw_dev mix test
|
||||
running build task
|
||||
...
|
||||
Finished in 0.02 seconds (0.02s async, 0.00s sync)
|
||||
1 doctest, 2 tests, 0 failures
|
||||
```
|
||||
|
||||
While it passes, this is not a valid test. `/public/index.html` already exists before our test runs the build task, because our `Dockerfile` tells Docker to run the task when it builds our image. To make the test valid, we need it to use a clean directory.
|
||||
|
||||
If we make the output directory configurable during runtime, we can create a temporary directory to use as a mock output directory during test runs. We need to move where the output directory is stored, from a hard-coded module attribute into something configureable at runtime.
|
||||
|
||||
## Separating `test` and `dev` environment data
|
||||
|
||||
Right now the value for the output directory is stored in a module attribute in the task module. If we had it stored in a variable, we could change it when we run our tests.
|
||||
|
||||
### Configuring the output directory
|
||||
|
||||
#### 1. Store the value in an application environment variable
|
||||
|
||||
To store it in an application environment variable, add this in `mix.exs`
|
||||
|
||||
```elixir
|
||||
def application do
|
||||
[
|
||||
env: [output_directory: "/public"],
|
||||
...
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
Then, in `run()` in `lib/mix/tasks/build.ex`, we fetch the output directory from the application environment instead of hard-coding it in a module attribute
|
||||
|
||||
```elixir
|
||||
def run(_args) do
|
||||
Mix.shell().info("running build task")
|
||||
|
||||
output_directory = Application.fetch_env!(:static_site, :output_directory)
|
||||
|
||||
File.mkdir_p!(output_directory)
|
||||
output_directory |> Path.join("index.html") |> File.write!(@markup)
|
||||
end
|
||||
```
|
||||
|
||||
Now that the output directory is stored in an application env var, let's see about changing it for our test run.
|
||||
|
||||
|
||||
#### 2. Change the value when running tests
|
||||
|
||||
Now we can dynamically set the output directory in our test. Really we just have to set it a relative path instead of an absolute. Once we make it a relative path, that mix task will create the output directory relative to the current directory.
|
||||
|
||||
```elixir
|
||||
setup do
|
||||
output_dir = "public"
|
||||
Application.put_env(:static_site, :output_directory, output_dir)
|
||||
|
||||
[output_dir: output_dir]
|
||||
end
|
||||
|
||||
|
||||
test "creates output file", %{output_path: output_dir} do
|
||||
...
|
||||
assert File.exists?(Path.join(output_dir, "index.html"))
|
||||
end
|
||||
```
|
||||
|
||||
#### 3. Restore the original value
|
||||
|
||||
But this changes the output directory application-wide, and it's good practice to put anything the tests change back to the way they were before. We can use `on_exit()` to set the original value when the test run finishes. We have to save it before changing the application variable and pass it to the call in `on_exit()`
|
||||
|
||||
```elixir
|
||||
setup do
|
||||
output_dir = "public"
|
||||
original = Application.fetch_env!(:static_site, :output_directory)
|
||||
...
|
||||
on_exit(fn ->
|
||||
Application.put_env(:static_site, :output_directory, original)
|
||||
end)
|
||||
...
|
||||
end
|
||||
```
|
||||
|
||||
Now that we can change the output directory in our tests, we can point to a temporary directory that is separate from our dev files. But where should we put it?
|
||||
|
||||
### Adding a temporary `tmp` directory for test artifacts
|
||||
|
||||
#### 1. Locate the temporary test directory
|
||||
|
||||
Before we spend too much time thinking about it, let's take a look how it's done in the Elixir source code. In [`lib/mix/test/test_helper.exs`](https://github.com/elixir-lang/elixir/blob/main/lib/mix/test/test_helper.exs) there's
|
||||
|
||||
```elixir
|
||||
def tmp_path do
|
||||
Path.expand("../tmp", __DIR__)
|
||||
end
|
||||
|
||||
def tmp_path(extension) do
|
||||
Path.join(tmp_path(), remove_colons(extension))
|
||||
end
|
||||
```
|
||||
|
||||
`__DIR__` is whatever directory the current file is in, so for our task test in `test/mix/tasks` the temp directory would be `test/mix/tmp`.
|
||||
|
||||
That seems like a good place to us. Let's copy `tmp_path()` into our test file. We can leave off the call to `remove_colons()`, since we don't have any colons to deal with.
|
||||
|
||||
```elixir
|
||||
defp tmp_path, do: Path.expand("../tmp", __DIR__)
|
||||
defp tmp_path(extension), do: Path.join(tmp_path(), extension)
|
||||
```
|
||||
|
||||
Now let's see about changing the location of the output directory.
|
||||
|
||||
#### 2. Change the current directory during the test
|
||||
|
||||
We can use `tmp_path/1` to create a temporary directory specifically for this test.
|
||||
|
||||
```elixir
|
||||
test "creates output file", %{output_dir: output_dir} do
|
||||
File.mkdir_p!(tmp_path("build"))
|
||||
File.cd!(tmp_path("build"))
|
||||
...
|
||||
```
|
||||
|
||||
And at the end of our test, we need to change back into the test file's directory with
|
||||
|
||||
```elixir
|
||||
...
|
||||
File.cd!(__DIR__)
|
||||
end
|
||||
```
|
||||
|
||||
#### 3. Add fixture for missing file
|
||||
|
||||
And since we're no longer in the project root directory, our task will not find the root `index.html` file. We need to add a fixture that our task can read from.
|
||||
|
||||
```elixir
|
||||
File.write("index.html", """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
""")
|
||||
```
|
||||
|
||||
After running the test,
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt hw_dev mix test
|
||||
running build task
|
||||
...
|
||||
Finished in 0.02 seconds (0.02s async, 0.00s sync)
|
||||
1 doctest, 2 tests, 0 failures
|
||||
```
|
||||
|
||||
we have our test directory, `test/mix/tmp`,
|
||||
|
||||
```
|
||||
$ find test/mix/tmp
|
||||
test/mix/tmp
|
||||
test/mix/tmp/build
|
||||
test/mix/tmp/build/public
|
||||
test/mix/tmp/build/public/index.html
|
||||
test/mix/tmp/build/index.html
|
||||
```
|
||||
|
||||
and our fixture file.
|
||||
|
||||
```
|
||||
$ cat test/mix/tmp/build/index.html
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
## Cleaning up temporary directories
|
||||
|
||||
Our tests are pretty solid now, but it is a good idea to clean the `tmp` directory up when the tests are done. We can do that in the `on_exit()` call in our test setup block
|
||||
|
||||
```elixir
|
||||
on_exit(fn ->
|
||||
...
|
||||
File.rm_rf!(tmp_path())
|
||||
end)
|
||||
```
|
||||
|
||||
Now, after we run the tests, all the files and directories created during the test run have been removed.
|
||||
|
||||
```
|
||||
$ find test/mix/tmp/
|
||||
find: ‘test/mix/tmp/’: No such file or directory
|
||||
```
|
||||
|
||||
|
||||
## Suppressing IO messages
|
||||
|
||||
Our mix task outputs a useful message when it runs, but we don't want that to clutter up the test results. We can prevent those messages by "capturing" the output with the `ExUnit.CaptureIO` module.
|
||||
|
||||
Add `import ExUnit.CaptureIO` to the top of our test file. Then, in our tests, we can pass our call to the mix task as a function to `capture_io()`
|
||||
|
||||
```elixir
|
||||
capture_io(fn -> Mix.Tasks.Build.run([]) end)
|
||||
```
|
||||
|
||||
When we run our tests now, there are no more IO messages cluttering up the test results.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt hw_dev mix test
|
||||
...
|
||||
Finished in 0.02 seconds (0.02s async, 0.00s sync)
|
||||
1 doctest, 2 tests, 0 failures
|
||||
```
|
||||
|
||||
|
||||
## Final form
|
||||
|
||||
Here's what the final draft of our test looks like.
|
||||
|
||||
::: filename-for-code-block
|
||||
`test/mix/tasks/build_test.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Mix.Tasks.BuildTest do
|
||||
use ExUnit.Case
|
||||
|
||||
import ExUnit.CaptureIO
|
||||
|
||||
setup do
|
||||
output_dir = "public"
|
||||
original = Application.fetch_env!(:static_site, :output_directory)
|
||||
Application.put_env(:static_site, :output_directory, output_dir)
|
||||
|
||||
on_exit(fn ->
|
||||
Application.put_env(:static_site, :output_directory, original)
|
||||
File.rm_rf!(tmp_path())
|
||||
end)
|
||||
|
||||
[output_dir: output_dir]
|
||||
end
|
||||
|
||||
test "creates output file", %{output_dir: output_dir} do
|
||||
File.mkdir_p!(tmp_path("build"))
|
||||
File.cd!(tmp_path("build"))
|
||||
|
||||
File.write("index.html", """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
capture_io(fn -> Mix.Tasks.Build.run([]) end)
|
||||
|
||||
assert File.exists?(Path.join(output_dir, "index.html"))
|
||||
|
||||
File.cd!(__DIR__)
|
||||
end
|
||||
|
||||
defp tmp_path, do: Path.expand("../tmp", __DIR__)
|
||||
defp tmp_path(extension), do: Path.join(tmp_path(), extension)
|
||||
end
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
|
||||
This test seemed trivial at first, but increased in complexity quickly. We had to set some of our configuration in application environment variables, change the configuration temporarily before a test run and then change it back after, clean up test artifacts after the run, and capture IO messages that were generated during it. That's enough going on that we thought it would be a good topic for a post. Cheers and happy coding!
|
||||
899
posts/2023-12-01-build-static-website-generator-part-1.md
Normal file
899
posts/2023-12-01-build-static-website-generator-part-1.md
Normal file
@@ -0,0 +1,899 @@
|
||||
---
|
||||
title: "Build A Static-Website Generator With Elixir, Part 1"
|
||||
blurb: "We take the first steps in designing and implementing the \"world's simplest static-website generator\". Building on tools and knowledge we acquired previously, and utilizing an incremental and iterative development process, we go through the entire software life-cycle from creating the initial project files to deploying to production. We spare nothing, from spelling out every command, to ensuring application integrity with tests, and even updating the README file. Grab a drink and some snacks, and dive right in!"
|
||||
...
|
||||
::: info
|
||||
This post was originally intended to be the first in a multi-part series.
|
||||
However, the deeper we got into this project, the more we realized we were
|
||||
basically implementing our own version of a web framework. Rather than
|
||||
continuing, we opted to change course and adopted an already-existing
|
||||
framework, [Phoenix](https://www.phoenixframework.org/), and simply added a
|
||||
Markdown-to-HTML conversion feature. As a consequence, there are no other parts
|
||||
to this post, but a description of the solution we chose instead can be found
|
||||
[here](/posts/publish-markdown-documents-as-static-web-pages-with-pandoc-and-phoenix).
|
||||
:::
|
||||
|
||||
This is one of our longer posts, so we've included a table of contents.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- Introduction
|
||||
- Incremental Development Concepts
|
||||
- Iterative Development Concepts
|
||||
- Initial Planning
|
||||
- Deliverables
|
||||
- Initialization
|
||||
- Planning
|
||||
- Requirements
|
||||
- Analysis & Design
|
||||
- Implementation
|
||||
- Testing
|
||||
- Deployment
|
||||
- Evaluation
|
||||
- Project Control List
|
||||
- New Tasks
|
||||
- Completed Tasks
|
||||
- Conclusion
|
||||
|
||||
## Introduction
|
||||
|
||||
We want to build the world's simplest static-website generator. Also, we want to use an iterative and incremental development method to do it. Finally, we want to use the Elixir programming language. Before we dive in, here's an extremely short primer on iterative and incremental development processes. We will combine these two ideas as we build this project.
|
||||
|
||||
### Incremental Development Concepts
|
||||
|
||||
- The system is broken down into small pieces
|
||||
- Requirements are worked through in order of priority
|
||||
|
||||
### Iterative Development Concepts
|
||||
|
||||
- The entire software development lifecycle is traversed in each iteration
|
||||
- Initialization, the initial "iteration", builds a base version of the system
|
||||
- All tasks and priorities are tracked in a Project Control List
|
||||
|
||||
We are now in the initialization phase. We will go through each of the seven iteration steps to build a minimum base system.
|
||||
|
||||
1. Planning
|
||||
2. Requirements
|
||||
3. Analysis & Design
|
||||
4. Implementation
|
||||
5. Testing
|
||||
6. Deployment
|
||||
7. Evaluation
|
||||
|
||||
## Initial Planning
|
||||
|
||||
### Deliverables
|
||||
|
||||
#### Functional Requirements
|
||||
|
||||
* A way to publish written work online (ie blog)
|
||||
|
||||
#### Non-Functional Requirements
|
||||
|
||||
* Must use Elixir
|
||||
|
||||
We love Elixir and functional programming.
|
||||
|
||||
* Must be as simple as possible
|
||||
|
||||
It's got to be dead simple. The app and the code. It's so easy to get carried away and over-engineer. Simplicity is a must-have requirement.
|
||||
|
||||
* Must use an iterative and incremental development method
|
||||
|
||||
In other words, short feedback loop/development life-cycle. Rather than releasing a minimum number of features as "version 1", we implement the smallest useful feature possible and release immediately.
|
||||
|
||||
## Initialization
|
||||
|
||||
### Planning
|
||||
|
||||
We want to deliver the smallest increment of functionality possible, so we're going to start simple. We will first add an `index.html` serves as the initial page for our static website. Then, we will add a development server to serve the site to ourselves so we can see how it looks before we deploy it live.
|
||||
|
||||
### Requirements
|
||||
|
||||
#### Functional
|
||||
|
||||
1. Post written text online
|
||||
|
||||
This is the essence of the deliverable. Posting written text online is the first functionality we will deliver.
|
||||
|
||||
#### Non-functional
|
||||
|
||||
* A way to see what the output will look like locally before it is deployed
|
||||
|
||||
This will make our lives easier during development and shorten the feedback loop.
|
||||
|
||||
### Analysis & Design
|
||||
|
||||
#### Analysis
|
||||
|
||||
The smallest bit of functionality we can offer is a static website with a single HTML file. We actually made a [proof of concept](https://github.com/webdevcat-me/docker_static_site_test) that does exactly this already and described the process in detail in an [earlier post](deploy-elixir-generated-html-with-docker-on-digitalocean.html). If we can add a development web server, that will pretty much meet all of our requirements.
|
||||
|
||||
#### Design
|
||||
|
||||
We could list our posts in a single HTML file, sorted in descending order of date, so that the most recent posts are at the top. Readers should have to scroll down to see older posts.
|
||||
|
||||
Ideally, each post would have a title, a short message, and the date it was posted. The text should not stretch across the whole viewport on a wide screen, but should wrap at a point that facilitates readability. The markup should be [well-structured](https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML/Document_and_website_structure) and valid. The column of text should be centered in the viewport but the text itself should be left-aligned.
|
||||
|
||||
##### Name
|
||||
|
||||
We've chosen "Serval" for the project name.
|
||||
|
||||
### Implementation
|
||||
|
||||
#### 1. Create a new Mix project
|
||||
|
||||
We're going to need a supervisor to start our development server later when we add it, so we use the `--sup` option when we run the `mix new` command to generate our initial project files. We can refer to our [explanation of this command](/posts/deploy-elixir-generated-html-with-docker-on-digitalocean.html#create-the-mix-project) previously, if necessary.
|
||||
|
||||
```
|
||||
$ docker run --rm -w /opt -v $PWD:/opt -u $(id -u):$(id -u) elixir mix new serval --sup
|
||||
```
|
||||
|
||||
Then, we initialize Git in our project directory.
|
||||
|
||||
```
|
||||
$ cd serval
|
||||
$ git init
|
||||
```
|
||||
|
||||
Configure the user name and email, if necessary.
|
||||
|
||||
```
|
||||
$ git config user.email "webdevcat@proton.me"
|
||||
$ git config user.name "Catalin Mititiuc"
|
||||
```
|
||||
|
||||
Lastly, create the initial commit.
|
||||
|
||||
```
|
||||
$ git add .
|
||||
$ git commit -m "Initial commit"
|
||||
```
|
||||
|
||||
#### 2. Create a GitHub repo
|
||||
|
||||
After creating the remote repo on GitHub, we add it as the origin and push up our initial commit.
|
||||
|
||||
```
|
||||
$ git remote add origin git@github.com:webdevcat-me/serval.git
|
||||
$ git push -u origin master
|
||||
```
|
||||
|
||||
#### 3. Add an `index.html` file to the root directory
|
||||
|
||||
Just a simple HTML boilerplate, for now.
|
||||
|
||||
::: filename-for-code-block
|
||||
`index.html`
|
||||
:::
|
||||
|
||||
```html
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hello, Serval!</h1>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
#### 4. Add a `Dockerfile`
|
||||
|
||||
As we've mentioned before, we heavily favor using Docker for our projects when possible. We can copy [the `Dockerfile` from our proof of concept app](https://github.com/webdevcat-me/docker_static_site_test/blob/master/Dockerfile), but since we added a new file, `index.html`, we need to add a `COPY` command for it as well.
|
||||
|
||||
::: filename-for-code-block
|
||||
`Dockerfile`
|
||||
:::
|
||||
|
||||
```dockerfile
|
||||
FROM elixir:slim
|
||||
|
||||
WORKDIR /opt
|
||||
COPY lib ./lib
|
||||
COPY mix.exs ./mix.exs
|
||||
COPY index.html ./index.html
|
||||
|
||||
ARG MIX_ENV=dev
|
||||
|
||||
RUN mix deps.get --only $MIX_ENV && mix build
|
||||
```
|
||||
|
||||
#### 5. Add a Mix task for building the site
|
||||
|
||||
We can start with a copy of `lib/mix/tasks/build.ex` [from our concept app](https://github.com/webdevcat-me/docker_static_site_test/blob/master/lib/mix/tasks/build.ex), as it already mostly does what we want. Our build task isn't really going to "build" anything, just yet. For now, it's just going to copy our `index.html` file from our app's root directory to the `/public` directory in the Docker container. We'll build on this later to make it more dynamic.
|
||||
|
||||
First, we want to add a variable for the output directory to the application environment. We do this in the `application()` function of the `mix.exs` file, as per the [documentation for the `mix compile.app` command](https://hexdocs.pm/mix/1.12/Mix.Tasks.Compile.App.html).
|
||||
|
||||
::: filename-for-code-block
|
||||
`mix.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
def application do
|
||||
[
|
||||
...
|
||||
env: [output_directory: "/public"],
|
||||
...
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
Next, after we copy over `lib/mix/tasks/build.ex` from our proof-of-concept app, we can remove the `@markup` module attribute because we want the output directory to contain the contents of the `index.html` file from our root directory.
|
||||
|
||||
Let's add a module attribute for our filename
|
||||
|
||||
```elixir
|
||||
@filename "index.html"
|
||||
```
|
||||
|
||||
and replace the last line in `run()` with
|
||||
|
||||
```elixir
|
||||
def run(_args) do
|
||||
...
|
||||
File.cp!(@filename, Path.join(output_directory, @filename))
|
||||
end
|
||||
```
|
||||
|
||||
And let's not forget to update the app name in the call to `fetch_env()`:
|
||||
|
||||
```elixir
|
||||
output_directory = Application.fetch_env!(:serval, :output_directory)
|
||||
```
|
||||
|
||||
Here's what the build task looks like. We also updated the `@moduledoc` to reflect the changes in the task, as well as updated and added shell output to inform us when the task finishes.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/mix/tasks/build.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Mix.Tasks.Build do
|
||||
@moduledoc "Creates a /public directory and copies an HTML index file from the root directory there"
|
||||
@shortdoc "Builds static HTML file"
|
||||
|
||||
use Mix.Task
|
||||
|
||||
@filename "index.html"
|
||||
|
||||
@impl Mix.Task
|
||||
def run(_args) do
|
||||
Mix.shell().info("Building markup...\n " <> @filename)
|
||||
|
||||
output_directory = Application.fetch_env!(:serval, :output_directory)
|
||||
|
||||
File.mkdir_p!(output_directory)
|
||||
File.cp!(@filename, Path.join(output_directory, @filename))
|
||||
|
||||
Mix.shell().info("Done.")
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
So far, everything is pretty much just like our concept app that we're using as a reference. To make sure there aren't any problems, let's try building our Docker image.
|
||||
|
||||
$ docker build -t serval_dev .
|
||||
|
||||
And now let's start a container and check the contents of the HTML file in `/public`.
|
||||
|
||||
$ docker run --rm -w /opt -v $PWD:/opt serval_dev bash -c "mix build && cat /public/index.html"
|
||||
Building markup...
|
||||
index.html
|
||||
Done.
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hello, Serval!</h1>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
So far, so good! Let's add these Docker commands to our `README.md` for future reference.
|
||||
|
||||
::: filename-for-code-block
|
||||
`README.md`
|
||||
:::
|
||||
|
||||
```markdown
|
||||
# Serval
|
||||
|
||||
The world's simplest static-website generator!
|
||||
|
||||
## Build Docker image
|
||||
|
||||
docker build -t serval_dev .
|
||||
|
||||
## Build static site
|
||||
|
||||
docker run --rm -w /opt -v $PWD:/opt serval_dev mix build
|
||||
|
||||
```
|
||||
|
||||
Since we expanded on the `build` task that we copied from our reference project, next we'll add a test for the new functionality.
|
||||
|
||||
#### 6. Add a test for our Mix build task
|
||||
|
||||
To start, our POC app has [a test for the Mix `build` task](https://github.com/webdevcat-me/docker_static_site_test/blob/master/test/mix/tasks/build_test.exs) we can copy over to our project in `test/mix/tasks/build_test.exs`. We'll want to add a new test case to cover the changes we've made.
|
||||
|
||||
```elixir
|
||||
test "updates output files"
|
||||
```
|
||||
|
||||
##### First pass
|
||||
|
||||
To test that our Mix task is updating the output files, we first need to create them so they exist already before the task runs. We need to create two fixtures in our test. The first is the file already existing in the output directory, and the second is the file in our root directory that will replace it when we run our build task.
|
||||
|
||||
```elixir
|
||||
test "updates output files", %{output_dir: output_dir} do
|
||||
output_path = Path.join(tmp_path("build"), output_dir)
|
||||
File.mkdir_p!(output_path)
|
||||
|
||||
File.write!(Path.join(output_path, "index.html"), """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
File.mkdir_p!(tmp_path("build"))
|
||||
File.cd!(tmp_path("build"))
|
||||
|
||||
File.write("index.html", """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>welcome</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
capture_io(fn -> Mix.Tasks.Build.run([]) end)
|
||||
|
||||
assert File.read!(Path.join(output_path, "index.html")) =~ "welcome"
|
||||
|
||||
File.cd!(__DIR__)
|
||||
end
|
||||
```
|
||||
|
||||
##### Refactoring
|
||||
|
||||
###### De-duplicate fixture
|
||||
|
||||
Both tests in our file now need to use an `index.html` fixture, so we can move that markup into a module attribute, `@markup`, to avoid duplicating it in both tests.
|
||||
|
||||
```elixir
|
||||
@markup """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
"""
|
||||
...
|
||||
test "creates output file", %{output_dir: output_dir} do
|
||||
...
|
||||
File.write("index.html", @markup)
|
||||
...
|
||||
end
|
||||
|
||||
test "updates output files", %{output_dir: output_dir} do
|
||||
...
|
||||
File.write!(Path.join(output_path, "index.html"), @markup)
|
||||
...
|
||||
end
|
||||
```
|
||||
|
||||
###### De-duplicate directory setup
|
||||
|
||||
Next, both our tests require setting up a temporary directory.
|
||||
|
||||
```elixir
|
||||
File.mkdir_p!(tmp_path("build"))
|
||||
File.cd!(tmp_path("build"))
|
||||
...
|
||||
File.cd!(__DIR__)
|
||||
```
|
||||
|
||||
The official Elixir code repository has [a test helper called `in_tmp()`](https://github.com/elixir-lang/elixir/blob/b829f1bae76f0fa847535083aef5e4e015ce0ab5/lib/mix/test/test_helper.exs#L122-L127) that does exactly this. We can copy that function into our test file. We're actually already using the function that it calls, `tmp_path()`, in our code. 😁
|
||||
|
||||
```elixir
|
||||
defp in_tmp(which, function) do
|
||||
path = tmp_path(which)
|
||||
File.rm_rf!(path)
|
||||
File.mkdir_p!(path)
|
||||
File.cd!(path, function)
|
||||
end
|
||||
```
|
||||
|
||||
And now we can remove the temp directory setup from our tests, and just wrap them in a call to `in_tmp()`.
|
||||
|
||||
```elixir
|
||||
test "creates output files", %{output_dir: output_dir} do
|
||||
in_tmp("build", fn ->
|
||||
...
|
||||
end)
|
||||
end
|
||||
|
||||
test "updates output files", %{output_dir: output_dir} do
|
||||
in_tmp("build", fn ->
|
||||
...
|
||||
end)
|
||||
end
|
||||
```
|
||||
|
||||
And since `in_tmp()` deletes any existing temp directory, we don't need to run our test teardown for every test. And since the output directory doesn't change between tests, we don't need to run our setup for every test, either. We can just do it all, once per test-run, by defining our setup block in a call to `setup_all()` instead of `setup()`.
|
||||
|
||||
```elixir
|
||||
setup_all do
|
||||
```
|
||||
|
||||
##### Cleanup
|
||||
|
||||
- We use the string `"build"` in multiple places, so we can move that into the setup block and pass it into our tests.
|
||||
- We call `Path.join(output_path, "index.html")` twice, so we can just call it once in the setup block and pass it to our tests.
|
||||
- We use the string `"index.html"` in multiple places. Let's put that in a module attribute `@filename`.
|
||||
|
||||
##### Final form
|
||||
|
||||
Here's what the final version of our test looks like.
|
||||
|
||||
::: filename-for-code-block
|
||||
`test/mix/tasks/build_test.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Mix.Tasks.BuildTest do
|
||||
use ExUnit.Case
|
||||
|
||||
import ExUnit.CaptureIO
|
||||
|
||||
@filename "index.html"
|
||||
|
||||
@markup """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>hello world</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
setup_all do
|
||||
output_dir = "public"
|
||||
file_path = Path.join(output_dir, @filename)
|
||||
|
||||
original = Application.fetch_env!(:serval, :output_directory)
|
||||
Application.put_env(:serval, :output_directory, "public")
|
||||
|
||||
on_exit(fn ->
|
||||
Application.put_env(:serval, :output_directory, original)
|
||||
File.rm_rf!(tmp_path())
|
||||
end)
|
||||
|
||||
[context: "build", output_dir: output_dir, file_path: file_path]
|
||||
end
|
||||
|
||||
test "creates output files", %{context: context, file_path: file_path} do
|
||||
in_tmp(context, fn ->
|
||||
File.write!(@filename, @markup)
|
||||
|
||||
capture_io(fn -> Mix.Tasks.Build.run([]) end)
|
||||
|
||||
assert File.exists?(file_path)
|
||||
end)
|
||||
end
|
||||
|
||||
test "updates output files", %{file_path: file_path} = context do
|
||||
in_tmp(context[:context], fn ->
|
||||
File.mkdir_p!(context[:output_dir])
|
||||
File.write!(file_path, @markup)
|
||||
|
||||
File.write!(@filename, """
|
||||
<!-- Auto-generated fixture -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>welcome</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
capture_io(fn -> Mix.Tasks.Build.run([]) end)
|
||||
|
||||
assert File.read!(file_path) =~ "welcome"
|
||||
end)
|
||||
end
|
||||
|
||||
defp tmp_path, do: Path.expand("../tmp", __DIR__)
|
||||
defp tmp_path(extension), do: Path.join(tmp_path(), extension)
|
||||
|
||||
defp in_tmp(which, function) do
|
||||
path = tmp_path(which)
|
||||
File.rm_rf!(path)
|
||||
File.mkdir_p!(path)
|
||||
File.cd!(path, function)
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
Let's try running our test.
|
||||
|
||||
$ docker run --rm -w /opt -v $PWD:/opt serval_dev mix test
|
||||
....
|
||||
Finished in 0.05 seconds (0.00s async, 0.05s sync)
|
||||
1 doctest, 3 tests, 0 failures
|
||||
|
||||
|
||||
And let's not forget to add this Docker command to the `README`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`README.md`
|
||||
:::
|
||||
|
||||
```markdown
|
||||
...
|
||||
## Run tests
|
||||
|
||||
docker run --rm -w /opt -v $PWD:/opt serval_dev mix test
|
||||
```
|
||||
|
||||
We still need to add some markup and styles to our `index` file, but without a web server to show us the content, making and verifying changes will be tedious. Let's now focus on the development server and then we'll come back around to markup and styling.
|
||||
|
||||
#### 7. Add a web server for development
|
||||
|
||||
To add our development web server, we can take a look at the Plug library documentation, as it provides a [nice example](https://hexdocs.pm/plug/readme.html#hello-world-request-response) to get us started.
|
||||
|
||||
We add `plug_cowboy` to `deps()` in our `mix.exs` file. Since the web server will only run during development, we use the `only: :dev` option.
|
||||
|
||||
```elixir
|
||||
defp deps do
|
||||
[
|
||||
{:plug_cowboy, "~> 2.0", only: :dev},
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
Next, we copy the example plug module from the Plug documentation to `lib/serval/plug.ex`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/serval/plug.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Serval.Plug do
|
||||
import Plug.Conn
|
||||
|
||||
def init(options) do
|
||||
# initialize options
|
||||
options
|
||||
end
|
||||
|
||||
def call(conn, _opts) do
|
||||
conn
|
||||
|> put_resp_content_type("text/plain")
|
||||
|> send_resp(200, "Hello world")
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
We can update `lib/serval/application.ex` to start our web server under a supervisor when the application starts. We'll also add a call to our build task to make sure we're not looking at a stale page on startup.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/serval/application.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Serval.Application do
|
||||
@moduledoc false
|
||||
|
||||
use Application
|
||||
require Logger
|
||||
|
||||
@impl true
|
||||
def start(_type, _args) do
|
||||
Mix.Tasks.Build.run([])
|
||||
|
||||
children = [
|
||||
{Plug.Cowboy, plug: Serval.Plug, scheme: :http, options: [port: 4000]}
|
||||
]
|
||||
|
||||
opts = [strategy: :one_for_one, name: Serval.Supervisor]
|
||||
|
||||
Logger.info("Plug now running on localhost:4000")
|
||||
|
||||
Supervisor.start_link(children, opts)
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
Now, we can test our development server. First, we fetch our new dependencies with:
|
||||
|
||||
$ docker run --rm -w /opt -v $PWD:/opt serval_dev mix deps.get
|
||||
|
||||
Then, we can start the application. Since we want our server to stay running until we stop it, we use the `--no-halt` option. We also need to publish a port, since our browser is connecting to a server inside a container. The `-p 4000:4000` option binds the container's port `4000` to port `4000` on our system.
|
||||
|
||||
$ docker run --rm -it -w /opt -v $PWD:/opt -p 4000:4000 serval_dev mix run --no-halt
|
||||
|
||||
It's going to ask to install `hex` and then `rebar`.
|
||||
|
||||
Mix requires the Hex package manager to fetch dependencies
|
||||
Shall I install Hex? (if running non-interactively, use "mix local.hex --force") [Yn]
|
||||
...
|
||||
Could not find "rebar3", which is needed to build dependency :telemetry
|
||||
Shall I install rebar3? (if running non-interactively, use "mix local.rebar --force") [Yn]
|
||||
...
|
||||
==> serval
|
||||
Compiling 4 files (.ex)
|
||||
Generated serval app
|
||||
Building markup...
|
||||
index.html
|
||||
Done.
|
||||
|
||||
01:26:30.509 [info] Plug now running on localhost:4000
|
||||
|
||||
And when we visit `localhost:4000`, we should see a "Hello world" message in plain text.
|
||||
|
||||
|
||||
|
||||
Great! So our web server is running and responding to requests! Before moving on, let's update our `README` with some new Docker commands.
|
||||
|
||||
::: filename-for-code-block
|
||||
`README.md`
|
||||
:::
|
||||
|
||||
```markdown
|
||||
...
|
||||
## Run development server
|
||||
|
||||
### First, fetch the dependencies
|
||||
|
||||
docker run --rm -w /opt -v $PWD:/opt serval_dev mix deps.get
|
||||
|
||||
### Start the server
|
||||
|
||||
docker run --rm -it -w /opt -v $PWD:/opt -p 4000:4000 serval_dev mix run --no-halt
|
||||
|
||||
Visit `localhost:4000` to view.
|
||||
```
|
||||
|
||||
#### 8. Serve files from the output directory
|
||||
|
||||
Our development web server currently responds to every call with "Hello world" in plain text. Now, lets see about making it serve our file from the container's `/public` directory.
|
||||
|
||||
We update `lib/serval/plug.ex` to get the output directory path from the application environment variable.
|
||||
|
||||
```elixir
|
||||
def call(conn, _opts) do
|
||||
path =
|
||||
:serval |> Application.fetch_env!(:output_directory) |> Path.join("index.html")
|
||||
...
|
||||
```
|
||||
|
||||
Then, we replace the call to `send_resp` with `send_file` and pass the MIME type `"text/html"` instead of `"text/plain"` to the `put_resp_content_type()` call.
|
||||
|
||||
```elixir
|
||||
...
|
||||
conn |> put_resp_content_type("text/html") |> send_file(200, path)
|
||||
end
|
||||
```
|
||||
|
||||
Now when we start the development server, we should see the contents of the `index.html` file from our project root.
|
||||
|
||||
|
||||
|
||||
If we make a change to `index.html`, we will not immediately see the results by refreshing the browser. We will first have to run our build task before we can see the new output. Since we added a call to the build task in `lib/serval/application.ex`, we could see the changes by simply restarting the dev server. To see the changes without restarting, we first need to find the name of the container running our app with `docker ps`. Then we can run a command in that container like so:
|
||||
|
||||
$ docker exec CONTAINER_NAME mix build
|
||||
|
||||
After that, refreshing the browser should show us any changes to `index.html`. That's also another one to add to `README.md`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`README.md`
|
||||
:::
|
||||
|
||||
```markdown
|
||||
...
|
||||
## View changes
|
||||
|
||||
To see updates made to `index.html`, simply restart the dev server. To see changes without restarting, first find the name of the running container with `docker ps`. Then, run:
|
||||
|
||||
docker exec CONTAINER_NAME mix build
|
||||
```
|
||||
|
||||
#### 9. Rebuild the Docker image
|
||||
|
||||
So, our dev server is running and it is serving the `index.html` file from our root directory. Awesome!
|
||||
|
||||
If you have stopped and started the dev server a few times, you will notice that it asks to install Hex each time. Because we always run our commands in a new container, the Hex installation doesn't get saved between stops and restarts. Since our `Dockerfile` fetches dependencies and calls the build task, all we need to do to fix the issue is rebuild our Docker image.
|
||||
|
||||
$ docker image rm serval_dev
|
||||
$ docker build -t serval_dev .
|
||||
|
||||
Now when we start our app, we will no longer receive the `Shall I install Hex?` prompt.
|
||||
|
||||
$ docker run --rm -it -w /opt -v $PWD:/opt -p 4000:4000 serval_dev mix run --no-halt
|
||||
Building markup...
|
||||
index.html
|
||||
Done.
|
||||
|
||||
03:24:57.061 [info] Plug now running on localhost:4000
|
||||
|
||||
#### 10. Avoid compiling the dev server in `:prod` (and running it in `:test`)
|
||||
|
||||
We don't require any dependencies in production, but we do compile code that still refers to those non-existent dependencies. We have to edit `mix.exs` to ensure that our web server code is not compiled and our application is not started in production.
|
||||
|
||||
```elixir
|
||||
def project do
|
||||
[
|
||||
...
|
||||
elixirc_paths: elixirc_paths(Mix.env()),
|
||||
...
|
||||
]
|
||||
end
|
||||
|
||||
def application do
|
||||
...
|
||||
mod: mod(Mix.env())
|
||||
]
|
||||
end
|
||||
|
||||
defp elixirc_paths(:prod), do: ["lib/mix/tasks"]
|
||||
defp elixirc_paths(_), do: ["lib"]
|
||||
|
||||
defp mod(:dev), do: {Serval.Application, []}
|
||||
defp mod(_), do: []
|
||||
```
|
||||
|
||||
#### 12. Update markup
|
||||
|
||||
Now that the dev server is working, viewing changes to our markup is much easier. We can go about improving what we have of our static website so far to make it more presentable.
|
||||
|
||||
Our design calls for "title, a short message, and the date". Since we are building the world's simplest static-website generator, the simplest thing we can think of is a heading, a paragraph, and a date:
|
||||
|
||||
We're going to keep it extremely simple. What is the smallest amount of information we can convey that is going to be useful? A page header would be good, so readers have something to identify us with. And then a heading, date, and a paragraph to satisfy our design.
|
||||
|
||||
```html
|
||||
<body>
|
||||
<header>
|
||||
<h1>Page Heading</h1>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h2>Post Heading</h2>
|
||||
<time datetime="2020-01-01">Jan 1, 2020</time>
|
||||
<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>
|
||||
</main>
|
||||
</body>
|
||||
```
|
||||
|
||||
After restarting our server, we can see our changes in the browser.
|
||||
|
||||
|
||||
|
||||
This is enough to get us started. Let's add some styling.
|
||||
|
||||
#### 13. Add styles
|
||||
|
||||
We're going to use white-space as much as possible do delineate different sections.
|
||||
|
||||
Styling is pretty subjective, and we're still early in this project, so we don't want to spend too much time on the appearance now. We just want to make the text a bit more readabale. We add our style directly to the head of the `index.html` file. We will break the styles out into their own file later, for now we just want to do the minimum to initialize the project base.
|
||||
|
||||
|
||||
Here's what we've added to the bottom of the `<head>` section in `index.html`:
|
||||
|
||||
```html
|
||||
...
|
||||
<link rel="stylesheet" href="//fonts.googleapis.com/css2?family=EB+Garamond&family=Pathway+Gothic+One" type="text/css" media="screen" />
|
||||
<style>
|
||||
body {
|
||||
font: 1.2em/1.5em 'EB Garamond', serif;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
body > main {
|
||||
max-width: 31.5em;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
body > header h1 {
|
||||
margin-top: 0;
|
||||
margin-bottom: 0.3em;
|
||||
}
|
||||
|
||||
h1, h2 {
|
||||
font-family: 'Pathway Gothic One', sans-serif;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
```
|
||||
|
||||
We can now see what our styles look like in a few different viewport widths. We want to make sure to test both super-narrow, for small mobile screens, and super-wide, for wide monitors.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Our goal is legibility, and that looks well-legible enough to get us started.
|
||||
|
||||
### Testing
|
||||
|
||||
Testing so far is pretty straight-forward: either our `index` page is viewable or it isn't. We're going to set up a staging environment to ensure that it's viewable once we deploy it.
|
||||
|
||||
#### Deploy to a staging environment
|
||||
|
||||
We already described our deployment process in [the earlier post](/posts/deploy-elixir-generated-html-with-docker-on-digitalocean.html) about the proof-of-concept for this project. We're going to follow the same steps to deploy Serval, but first we want to create a separate resource for staging, so that we can look everything over one last time before we commit to deploying for production.
|
||||
|
||||
One difference from our POC instructions is, after we add our GitHub repository, DigitalOcean detects two resources: a Static Site and a Web Service. We want to delete the the Static Site resource and continue with the deployment steps using the resource that was detected as a Web Service.
|
||||
|
||||
1. For our Resource Name, we used, simply, `staging`.
|
||||
2. We also want to set an HTTP Request Route for something like `/staging`, because the root route `/` will be for production.
|
||||
3. On the info screen, we change the default app name to `serval` and click "Next".
|
||||
4. Then we finish up by clicking "Create Resources".
|
||||
|
||||
If all goes well, when deploying finishes, we should be able to visit the `/staging` route of the URL DigitalOcean gives for our app. Something like:
|
||||
|
||||
https://serval-abc12.ondigitalocean.app/staging
|
||||
|
||||
If everything looks alright, we can tag the release and proceed to deploying a production version.
|
||||
|
||||
#### Tag the release
|
||||
|
||||
And that's it! With our requirements met, we are ready to release. Let's add a tag to our repository to mark the release version and push it up to our remote repo.
|
||||
|
||||
$ git tag v0.1.0
|
||||
$ git push origin v0.1.0
|
||||
|
||||
### Deployment
|
||||
|
||||
#### Deploy to a production environment
|
||||
|
||||
To deploy our production version of Serval, we're going to create a new resource for the DigitalOcean App we just created.
|
||||
|
||||
1. Go to the DigitalOcean Apps page.
|
||||
2. Click on the Serval App. We named ours, appropriately, `serval`.
|
||||
3. Click on "Create" and select "Create Resources from Source Code" from the menu.
|
||||
4. As before, select our Serval repository.
|
||||
5. As with our staging process above, it will detect two resources. We want to delete the Static Site one and edit the Web Service one as before.
|
||||
6. We set our resource name as `production`.
|
||||
7. Change the resource type from Web Service to Static Site and click "Save".
|
||||
8. Again, we want to set our output directory to `/public` and click "Save".
|
||||
9. Make sure we add a root route, `/`, under HTTP Request Routes and click "Save".
|
||||
10. If we added a `MIX_ENV` with a value of `prod` under Global when deploying the staging resource, the we won't have to do anything else on the Environment Variables screen. Otherwise, we should make sure we add it, either under Global or under "production".
|
||||
11. And if everything looks alright on the Review screen, we click "Create Resource".
|
||||
|
||||
Once deployment finishes, our App will have two components: a "staging" one and a "production" one. We can now visit our production app at the URL DigitalOcean provides (or a custom URL, but that is outside the scope of this post).
|
||||
|
||||
### Evaluation
|
||||
|
||||
We'll now take a moment to consider our project so far. On the functional side, what's most obvious is that we're not really "generating" anything, we're still just editing raw HTML. We'll adress this point soon.
|
||||
|
||||
On the non-functional side, two things in particular stick out in our minds.
|
||||
|
||||
1. Having to run the build task to see a development change is repetitive and time-consuming
|
||||
2. Having to refresh the browser to see a development change is, again, repetitive and time-consuming
|
||||
|
||||
The consequences of having to do these repetitive steps over and over are a slower feedback loop and a longer development cycle. Luckily, these sorts of issues are easily remedied with a little automation. That sounds like a great next-step for this project and a great addition to our Project Control List.
|
||||
|
||||
## Project Control List
|
||||
|
||||
Here's what we've accomplished in this iteration, and what we look forward to working on in the next one.
|
||||
|
||||
### New Tasks
|
||||
|
||||
1. Automate running the build task after code changes
|
||||
2. Automate refreshing the browser after a rebuild
|
||||
|
||||
### Completed Tasks
|
||||
|
||||
And here's what we've accomlished in this initial project step, building on the proof-of-concept we built before.
|
||||
|
||||
- ~~Add a development web server~~
|
||||
- ~~Add a root `index.html` file~~
|
||||
|
||||
## Conclusion
|
||||
|
||||
We have done the initialization phase and planted the "seed" from which the rest of our app will evolve. So far, we can publish a website with a single page and we can run a web server in our development environment to serve it to ourselves locally. Next time, we will make the build task run automatically whenever our markup changes. We'll see each other again, then. Take care and happy coding!
|
||||
152
posts/2024-01-01-build-a-neovim-qt-appimage-from-source.md
Normal file
152
posts/2024-01-01-build-a-neovim-qt-appimage-from-source.md
Normal file
@@ -0,0 +1,152 @@
|
||||
---
|
||||
title: "Build A Neovim Qt AppImage from Source"
|
||||
blurb: "Building an AppImage package from source allows us to run the
|
||||
latest version of Neovim-QT on our machine running the Debian Linux
|
||||
distribution."
|
||||
...
|
||||
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
We have [Debian](https://www.debian.org/) installed on our machine and would like to run [Neovim](https://neovim.io/) with the latest version of the [Neovim Qt](https://github.com/equalsraf/neovim-qt) GUI. While a Debian `neovim-qt` package exists, it is not the latest version. To solve this problem, we will build Neovim Qt from source and package it as an AppImage.
|
||||
|
||||
### Requirements
|
||||
|
||||
We will assume Neovim is already installed. Neovim AppImages are available from the [Neovim GitHub repo](https://github.com/neovim/neovim/releases).
|
||||
|
||||
## 1. Install [Toolbox](https://containertoolbx.org/)
|
||||
|
||||
We will have to install all the Neovim Qt build dependencies, so we will use Toolbox to build in a container, keeping our system clean.
|
||||
|
||||
$ sudo apt-get update
|
||||
$ sudo apt-get install podman-toolbox
|
||||
|
||||
## 2. Download Neovim Qt
|
||||
|
||||
We start by downloading the latest `*.tar.gz` source code asset from the [Neovim Qt GitHub repository](https://github.com/equalsraf/neovim-qt/releases/).
|
||||
|
||||
Then, we unpack and unzip it and `cd` into the directory.
|
||||
|
||||
$ tar -xzvf neovim-qt-0.2.18.tar.gz
|
||||
$ cd neovim-qt-0.2.18
|
||||
|
||||
## 3. Create and enter a new Toolbox container
|
||||
|
||||
$ toolbox create --distro debian neovim-qt
|
||||
$ toolbox enter neovim-qt
|
||||
|
||||
## 4. Add `deb-src` to the sources list
|
||||
|
||||
Toolbox's base Debian image only lists binary archive types in the sources list. We will have to add the archive type for source packages, as well.
|
||||
|
||||
$ sudo nano /etc/apt/sources.list.d/debian.sources
|
||||
|
||||
We change the two lines that read `Types: deb` to `Types: deb deb-src` and save the changes.
|
||||
|
||||
Types: deb deb-src
|
||||
...
|
||||
|
||||
Types: deb deb-src
|
||||
...
|
||||
|
||||
## 5. Install build dependencies
|
||||
|
||||
We can install all the build dependencies we will need to build Neovim Qt with the `build-dep` option. `fuse` will be needed to build the AppImage package.
|
||||
|
||||
$ sudo apt-get update
|
||||
$ sudo apt-get build-dep neovim-qt
|
||||
$ sudo apt-get install fuse
|
||||
|
||||
## 6. Add a build script
|
||||
|
||||
We copy a sample `cmake` build-script from the [appimage.org online documentation](https://docs.appimage.org/packaging-guide/from-source/native-binaries.html#bundle-qtquickapp-with-cmake) into a file called `build-with-cmake.sh`.
|
||||
|
||||
Alternatively, we can download it directly from their GitHub repository with
|
||||
|
||||
$ wget https://raw.githubusercontent.com/linuxdeploy/QtQuickApp/master/travis/build-with-cmake.sh
|
||||
|
||||
We need to make two changes.
|
||||
|
||||
1. On the line that contains
|
||||
|
||||
cmake "$REPO_ROOT" -DCMAKE_INSTALL_PREFIX=/usr
|
||||
|
||||
we add the variable `DCMAKE_BUILD_TYPE` and set it to `Release` (per the [Neovim Qt build instructions](https://github.com/equalsraf/neovim-qt/wiki/Build-Instructions)):
|
||||
|
||||
cmake "$REPO_ROOT" -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release
|
||||
|
||||
2. On the very last line,
|
||||
|
||||
mv QtQuickApp*.AppImage "$OLD_CWD"
|
||||
|
||||
we remove the sample app name, `QtQuickApp`.
|
||||
|
||||
mv *.AppImage "$OLD_CWD"
|
||||
|
||||
(We could, optionally, set it to `mv Neovim-Qt*.AppImage "$OLD_CWD"`, but for our case, it's not necessary).
|
||||
|
||||
## 7. Run the build script
|
||||
|
||||
We make the script runnable and then run it.
|
||||
|
||||
$ chmod +x build-with-cmake.sh
|
||||
$ ./build-with-cmake.sh
|
||||
|
||||
## 8. Test-run the AppImage
|
||||
|
||||
We should now have an AppImage package in our directory that we can run.
|
||||
|
||||
$ ./Neovim-Qt-x86_64.AppImage
|
||||
|
||||
When we run it we should see Neovim open in a new GUI window.
|
||||
|
||||
|
||||
|
||||
## 9. Exit the Toolbox container
|
||||
|
||||
$ exit
|
||||
logout
|
||||
|
||||
The Toolbox container is still running. We can stop it with
|
||||
|
||||
$ podman stop neovim-qt
|
||||
|
||||
## 10. Add the package to our user-specific executable directory
|
||||
|
||||
According to the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html), user-specific executables belong in `$HOME/.local/bin`. We will place our AppImage in its own directory in `~/.local/neovim-qt` and create a symlink to it in `~/.local/bin`.
|
||||
|
||||
$ mkdir ~/.local/neovim-qt
|
||||
$ mv ./Neovim-Qt-x86_64.AppImage ~/.local/neovim-qt
|
||||
$ ln -s ~/.local/neovim-qt/Neovim-Qt-x86_64.AppImage ~/.local/bin/nvim-qt
|
||||
|
||||
We can now run it by calling `nvim-qt` directly from the command line.
|
||||
|
||||
$ nvim-qt --version
|
||||
NVIM-QT v0.2.18.0
|
||||
Build type: Release
|
||||
Compilation: -Wall -Wextra -Wno-unused-parameter -Wunused-variable
|
||||
Qt Version: 5.15.8
|
||||
...
|
||||
|
||||
## 11. Add Neovim Qt to the applications menu
|
||||
|
||||
We simply need to copy the `.desktop` file from the source directory.
|
||||
|
||||
$ cp src/gui/nvim-qt.desktop ~/.local/share/applications/
|
||||
|
||||
## 12. Add an icon
|
||||
|
||||
And finally, we copy over the icon from the source directory as well.
|
||||
|
||||
$ mkdir ~/.local/neovim-qt/icons
|
||||
$ cp third-party/neovim.png ~/.local/neovim-qt/icons/nvim-qt.png
|
||||
$ mkdir -p ~/.local/share/icons/hicolor/192x192/apps
|
||||
$ ln -s ~/.local/neovim-qt/icons/nvim-qt.png ~/.local/share/icons/hicolor/192x192/apps/
|
||||
$ xdg-icon-resource forceupdate --mode user
|
||||
|
||||
## Conclusion
|
||||
|
||||
When we search for `neovim` in our applications menu, we should now see an entry we can use to start our new Neovim Qt AppImage.
|
||||
|
||||
|
||||
175
posts/2024-11-03-set-up-a-gitweb-server.md
Normal file
175
posts/2024-11-03-set-up-a-gitweb-server.md
Normal file
@@ -0,0 +1,175 @@
|
||||
---
|
||||
title: "Set Up A GitWeb Server"
|
||||
blurb: "Set up a VPS with a simple, web-based code repository visualizer using
|
||||
Lighttpd and GitWeb."
|
||||
...
|
||||
## Introduction
|
||||
|
||||
Git comes with a CGI script called GitWeb, a simple web-based visualizer.
|
||||
Today we will set up a virtual server on DigitalOcean that will use GitWeb to
|
||||
visualize a Git repository.
|
||||
|
||||
## 1. Create a DigitalOcean droplet
|
||||
|
||||
Create a new droplet from the DigitalOcean dashboard. For now, we go with the
|
||||
smallest virtual server currently available. We also add an SSH key so we can
|
||||
authenticate without a password.
|
||||
|
||||
|
||||
|
||||
## 2. Log in to the droplet remotely
|
||||
|
||||
After the droplet is created, we can see its IP address on the dashboard. We
|
||||
use this IP address to log in to our virtual server:
|
||||
|
||||
$ ssh root@XX.XX.XXX.XXX
|
||||
|
||||
## 3. Silence locale warnings
|
||||
|
||||
After successfully logging in, one of the messages we are greeted with is a
|
||||
warning about the locale.
|
||||
|
||||
-bash: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8)
|
||||
|
||||
We can make this annoying message go away by creating a locale file manually:
|
||||
|
||||
# localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||
|
||||
## 4. Add a user account to administer the Git repositories
|
||||
|
||||
Next, we create a user account that will administer the git repositories, so we
|
||||
don't always have to do it as `root`.
|
||||
|
||||
# adduser git --disabled-password
|
||||
|
||||
We have to add our SSH key to the `git` user's `authorized_keys` file.
|
||||
|
||||
# su git
|
||||
$ cd
|
||||
$ mkdir .ssh && chmod 700 .ssh
|
||||
$ touch .ssh/authorized_keys && chmod 600 .ssh/authorized_keys
|
||||
$ exit
|
||||
# cat ~/.ssh/authorized_keys >> /home/git/.ssh/authorized_keys
|
||||
|
||||
## 5. Install necessary packages
|
||||
|
||||
Lighttpd is the default web server that GitWeb tries to use if available. For
|
||||
simplicity, that's what we'll use.
|
||||
|
||||
# apt-get update
|
||||
# apt-get install git lighttpd gitweb
|
||||
|
||||
## 6. Configure Lighttpd
|
||||
|
||||
We will need to make some changes to Lighttpd's config file.
|
||||
|
||||
# nano /etc/lighttpd/lighttpd.conf
|
||||
|
||||
1. Update the value for `server.document-root`:
|
||||
|
||||
`server.document-root = "/usr/share/gitweb"`
|
||||
|
||||
2. Add `index.cgi` to `index-file.names`:
|
||||
|
||||
`index-file.names = ( "index.php", "index.html", "index.cgi" )`
|
||||
|
||||
## 7. Enable Lighttpd modules
|
||||
|
||||
Since GitWeb uses CGI, we will have to enable Lighttpd's CGI module. We will
|
||||
also need the setenv module. First we need to configure them.
|
||||
|
||||
# nano /etc/lighttpd/conf-available/05-setenv.conf
|
||||
|
||||
Add the following line:
|
||||
|
||||
::: filename-for-code-block
|
||||
`/etc/lighttpd/conf-available/05-setenv.conf`
|
||||
:::
|
||||
|
||||
...
|
||||
setenv.add-environment = ( "PATH" => env.PATH, "GITWEB_CONFIG" => env.GITWEB_CONFIG )
|
||||
|
||||
Next, edit the CGI module config file.
|
||||
|
||||
# nano /etc/lighttpd/conf-available/10-cgi.conf
|
||||
|
||||
Add:
|
||||
|
||||
::: filename-for-code-block
|
||||
`/etc/lighttpd/conf-available/10-cgi.conf`
|
||||
:::
|
||||
|
||||
...
|
||||
cgi.assign = (
|
||||
".cgi" => ""
|
||||
)
|
||||
|
||||
Once that's done, enable `mod_cgi` and `mod_setenv` with:
|
||||
|
||||
# lighty-enable-mod cgi setenv
|
||||
# service lighttpd force-reload
|
||||
|
||||
## 8. Edit the Lighttpd service init files
|
||||
|
||||
We need to edit the Lighttpd service startup files to define the
|
||||
`GITWEB_CONFIG` environment variable that we used in the previous step.
|
||||
|
||||
# systemctl edit lighttpd.service
|
||||
|
||||
This will start up an editor and create an `override.conf` file. Add the
|
||||
following two lines:
|
||||
|
||||
::: filename-for-code-block
|
||||
`/etc/systemd/system/lighttpd.service.d/override.conf`
|
||||
:::
|
||||
|
||||
[Service]
|
||||
Environment="GITWEB_CONFIG=./gitweb_config.perl"
|
||||
|
||||
Then, save the file and exit the editor. To finish, we need to run these two
|
||||
commands:
|
||||
|
||||
# systemctl daemon-reload
|
||||
# service lighttpd restart
|
||||
|
||||
## 9. Upload a Git repository
|
||||
|
||||
We are now ready to upload a Git respository we wish to visualize with our
|
||||
server. First, lets transfer ownership of the respository directory to our user
|
||||
`git` we created earlier.
|
||||
|
||||
# chown git /var/lib/git/
|
||||
# chgrp git /var/lib/git/
|
||||
|
||||
Now we can log out of the server with `exit`. On our local machine, we clone a
|
||||
`bare` copy of the repo we want to upload.
|
||||
|
||||
$ git clone --bare my_project my_project.git
|
||||
|
||||
Now we can upload this bare repo to our server.
|
||||
|
||||
$ scp -r my_project.git git@XX.XX.XXX.XXX:/var/lib/git
|
||||
|
||||
We can tell Git to automatically add group write permissions to our repo with:
|
||||
|
||||
$ ssh git@XX.XX.XXX.XXX
|
||||
$ cd /var/lib/git/my_project.git
|
||||
$ git init --bare --shared
|
||||
|
||||
## 10. Visit the GitWeb server
|
||||
|
||||
When we visit the server's IP address with our browser, `http://XX.XX.XXX.XXX`,
|
||||
we should see a GitWeb `projects` page. We can now explore our project's code
|
||||
with our web browser.
|
||||
|
||||
|
||||
|
||||
## Resources
|
||||
|
||||
Here is a list of the resources that were used to figure out how to accomplish
|
||||
the task in this post.
|
||||
|
||||
- [Git Documentation Book - Chapter 4: Git on the Server](https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols)
|
||||
- [Lighttpd Configuration Tutorial](https://redmine.lighttpd.net/projects/lighttpd/wiki/TutorialConfiguration)
|
||||
- [LinuxQuestions.org - Perl CGI:Can't locate CGI.pm](https://www.linuxquestions.org/questions/programming-9/perl-cgi-can%27t-locate-cgi-pm-330706/)
|
||||
- [Nicketa's GitHub Gist - LC_CTYPE.md](https://gist.github.com/nicks9188/a19f39d62780055a68c22b89a9799c25)
|
||||
41
posts/2024-11-11-resize-a-qemu-disk-image.md
Normal file
41
posts/2024-11-11-resize-a-qemu-disk-image.md
Normal file
@@ -0,0 +1,41 @@
|
||||
---
|
||||
title: "Resize A QEMU Disk Image"
|
||||
blurb: "Our hosting provider allows us to upload a custom image when
|
||||
provisioning a new server. We will have to resize the image if it does not
|
||||
match the size of the server's disk space."
|
||||
...
|
||||
## 1. Install `guestfs-tools`
|
||||
|
||||
$ apt-get update
|
||||
$ apt-get install guestfs-tools
|
||||
|
||||
## 2. Download an image
|
||||
|
||||
Download an image from [https://cdimage.debian.org/images/cloud/](https://cdimage.debian.org/images/cloud/).
|
||||
|
||||
## 3. List partitions
|
||||
|
||||
Check the image disk partitions to see which partition needs resizing.
|
||||
|
||||
$ virt-filesystems --long -h --all -a debian-12-nocloud-amd64.qcow2
|
||||
|
||||
## 4. Create output container
|
||||
|
||||
$ qemu-img create -f qcow2 \
|
||||
-o preallocation=metadata debian-12-nocloud-amd64-resized.qcow2 20G
|
||||
|
||||
## 5. Generate resized image
|
||||
|
||||
$ virt-resize --expand /dev/sda1 \
|
||||
debian-12-nocloud-amd64.qcow2 debian-12-nocloud-amd64-resized.qcow2
|
||||
|
||||
## Conclusion
|
||||
|
||||
You can now open `debian-12-nocloud-amd64-resized.qcow2` with a virtual machine
|
||||
program and the disk size will be 20G.
|
||||
|
||||
## References
|
||||
|
||||
`virt-resize` manpage
|
||||
|
||||
$ man virt-resize
|
||||
@@ -0,0 +1,540 @@
|
||||
---
|
||||
title: "Publish Markdown Documents As Static Web Pages with Pandoc and Phoenix"
|
||||
blurb: "We thought we wanted a static website generator. It turns out what we
|
||||
really wanted was Phoenix, with an option to convert markdown to HTML. Here is
|
||||
our implementation of a solution, using our very own, recently-released,
|
||||
Pandoc Hex package!"
|
||||
...
|
||||
|
||||
## Introduction
|
||||
|
||||
A short while ago, we published our latest version of a [`pandoc` installer Hex
|
||||
package](https://hex.pm/packages/pandoc), modeled after the existing
|
||||
[`tailwind`](https://hex.pm/packages/tailwind) and
|
||||
[`esbuild`](https://hex.pm/packages/esbuild) packages. In this post, we will
|
||||
show how we used our `pandoc` package to add our current markdown publishing
|
||||
solution to the Phoenix Framework.
|
||||
|
||||
## 1. Generate a new Phoenix project and add the `pandoc` dependency
|
||||
|
||||
Let's start with a new project. We will use the `--no-ecto` option because we
|
||||
don't need a database for this project.
|
||||
|
||||
$ mix phx.new hello --no-ecto
|
||||
|
||||
Next we add `pandoc` as a dependency to our `mix.exs` file.
|
||||
|
||||
::: filename-for-code-block
|
||||
`mix.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
{:pandoc, "~> 0.3", only: :dev}
|
||||
```
|
||||
|
||||
Then we fetch our dependencies.
|
||||
|
||||
$ mix deps.get
|
||||
|
||||
## 2. Configure `pandoc`
|
||||
|
||||
Because the goal is to have multiple documents, the name of the output file
|
||||
will depend on the name of the input file. For this reason, we cannot simply
|
||||
use a static value for the `--output` option. To deal with this problem,
|
||||
`pandoc` accepts a function for the `args` config key that allows us to set the
|
||||
output filename dynamically for each document.
|
||||
|
||||
::: filename-for-code-block
|
||||
`config/config.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
if config_env() != :prod do
|
||||
# Configure pandoc (the version is required)
|
||||
config :pandoc,
|
||||
version: "3.6.1",
|
||||
hello: [
|
||||
args: fn extra_args ->
|
||||
{_, [input_file], _} = OptionParser.parse(extra_args, switches: [])
|
||||
~w(--output=../priv/static/posts/#{Path.rootname(input_file)}.html)
|
||||
end,
|
||||
cd: Path.expand("../documents", __DIR__)
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
Because [anonymous functions are not supported in
|
||||
releases](https://elixirforum.com/t/mix-do-compile-release-could-not-read-configuration-file-config-runtime-exs-not-found/37800/3),
|
||||
we can wrap our config in a `if config_env() != :prod` conditional, since we'll
|
||||
only convert markdown to HTML at build time.
|
||||
|
||||
Next, we create the directory where our converted HTML documents will live.
|
||||
|
||||
$ mkdir priv/static/posts
|
||||
|
||||
And we add our new directory to `.gitignore` so that Git doesn't save the HTML
|
||||
output of our documents.
|
||||
|
||||
::: filename-for-code-block
|
||||
`.gitignore`
|
||||
:::
|
||||
|
||||
```
|
||||
# Ignore documents that are produced by pandoc.
|
||||
/priv/static/posts/
|
||||
```
|
||||
|
||||
Lastly, we add our `pandoc` watcher to the endpoint so that, in development,
|
||||
any changes to our documents' markdown will reflect in the browser in
|
||||
real-time.
|
||||
|
||||
::: filename-for-code-block
|
||||
`config/dev.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
config :hello, HelloWeb.Endpoint,
|
||||
# ...
|
||||
watchers: [
|
||||
# ...
|
||||
pandoc: {Pandoc, :watch, [:hello]}
|
||||
]
|
||||
```
|
||||
|
||||
To make sure everything is working, we can give it a quick test:
|
||||
|
||||
```bash
|
||||
$ mkdir documents
|
||||
$ echo "# hello" > documents/hello.md
|
||||
$ mix pandoc hello hello.md
|
||||
... [debug] Downloading pandoc from ...
|
||||
$ cat priv/static/posts/hello.html
|
||||
<h1 id="hello">hello</h1>
|
||||
```
|
||||
|
||||
## 3. Add new document aliases to `mix.exs`
|
||||
|
||||
Now that we have Pandoc installed, and a Mix task to convert a document from
|
||||
markdown to HTML, we need a way to call it on all the documents in the
|
||||
`documents` directory. We can do this by adding a new alias in `mix.exs` that
|
||||
will scan the directory for files and call the Pandoc Mix task on each.
|
||||
|
||||
::: filename-for-code-block
|
||||
`mix.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defp aliases do
|
||||
[
|
||||
setup: [
|
||||
"deps.get",
|
||||
"assets.setup",
|
||||
"assets.build",
|
||||
"documents.setup",
|
||||
"documents.build"
|
||||
],
|
||||
# ...
|
||||
"documents.setup": ["pandoc.install --if-missing"],
|
||||
"documents.build": &pandoc/1,
|
||||
"statics.deploy": ["assets.deploy", "documents.build"]
|
||||
]
|
||||
end
|
||||
|
||||
defp pandoc(_) do
|
||||
config = Application.get_env(:pandoc, :hello)
|
||||
cd = config[:cd] || File.cwd!()
|
||||
|
||||
cd
|
||||
|> File.cd!(fn ->
|
||||
Enum.filter(File.ls!(), &(File.stat!(&1).type != :directory))
|
||||
end)
|
||||
|> Enum.each(&Mix.Task.rerun("pandoc", ["hello", &1]))
|
||||
end
|
||||
```
|
||||
|
||||
Now when we run `mix setup`, Pandoc will convert all the files in our documents
|
||||
directory to markup and place the output in `priv/static/posts`.
|
||||
|
||||
We also added a `statics.deploy` alias so we'll only have to run a single task
|
||||
before we build a release.
|
||||
|
||||
## 4. Add context, controller, templates, and routes
|
||||
|
||||
Now that we have our documents in `priv/static/posts` in HTML format, we need a
|
||||
way to render them.
|
||||
|
||||
We start with a struct that will hold the values for each post.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello/documents/post.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Hello.Documents.Post do
|
||||
defstruct [:id, :path, :body]
|
||||
end
|
||||
```
|
||||
|
||||
Next, we add our Documents context that will be responsible for fetching a list
|
||||
of all posts as well as each individual post.
|
||||
|
||||
We have chosen to name our documents using hyphens (`-`) to separate words. Our
|
||||
post ids, then, will be the document filename minus the file extension suffix.
|
||||
So, a file `documents/this-is-the-first-post.md` will have an id of
|
||||
`this-is-the-first-post` and a URI that looks like
|
||||
`https://example.org/posts/this-is-the-first-post`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello/documents.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule Hello.Documents do
|
||||
@moduledoc """
|
||||
The Documents context.
|
||||
"""
|
||||
|
||||
alias Hello.Documents.Post
|
||||
|
||||
def list_posts do
|
||||
"documents/*"
|
||||
|> Path.wildcard()
|
||||
|> Enum.map(fn path ->
|
||||
%Post{
|
||||
id: path |> Path.rootname() |> Path.basename(),
|
||||
path: path
|
||||
}
|
||||
end)
|
||||
end
|
||||
|
||||
def get_post!(id) do
|
||||
post = Enum.find(list_posts(), fn post -> post.id == id end)
|
||||
|
||||
body =
|
||||
:hello
|
||||
|> :code.priv_dir()
|
||||
|> Path.join("static/posts/#{id}.html")
|
||||
|> File.read!()
|
||||
|
||||
%{post | body: body}
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
Our posts controller and view are pretty standard.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/controllers/post_controller.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule HelloWeb.PostController do
|
||||
use HelloWeb, :controller
|
||||
|
||||
alias Hello.Documents
|
||||
|
||||
def index(conn, _params) do
|
||||
posts = Documents.list_posts()
|
||||
render(conn, :index, posts: posts)
|
||||
end
|
||||
|
||||
def show(conn, %{"id" => id}) do
|
||||
post = Documents.get_post!(id)
|
||||
render(conn, :show, post: post)
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/controllers/post_html.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
defmodule HelloWeb.PostHTML do
|
||||
use HelloWeb, :html
|
||||
|
||||
embed_templates "post_html/*"
|
||||
end
|
||||
```
|
||||
|
||||
Our `index` and `show` templates are pretty similar to what the Phoenix
|
||||
`phx.gen.html` HTML generator spits out. We take full advantage of the UI core
|
||||
components that Phoenix provides.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/controllers/post_html/index.html.heex`
|
||||
:::
|
||||
|
||||
```heex
|
||||
<.header>
|
||||
Listing Posts
|
||||
</.header>
|
||||
|
||||
<.table id="posts" rows={@posts} row_click={&JS.navigate(~p"/posts/#{&1}")}>
|
||||
<:col :let={post} label="id"><%= post.id %></:col>
|
||||
<:action :let={post}>
|
||||
<div class="sr-only">
|
||||
<.link navigate={~p"/posts/#{post}"}>Show</.link>
|
||||
</div>
|
||||
</:action>
|
||||
</.table>
|
||||
```
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/controllers/post_html/show.html.heex`
|
||||
:::
|
||||
|
||||
```heex
|
||||
<.header>
|
||||
<%= @post.id %>
|
||||
<:subtitle>This is a post from your markdown documents.</:subtitle>
|
||||
</.header>
|
||||
|
||||
<%= raw(@post.body) %>
|
||||
|
||||
<.back navigate={~p"/posts"}>Back to posts</.back>
|
||||
```
|
||||
|
||||
Finally, we add the routes we will need, with only the `index` and `show`
|
||||
actions being necessary.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/router.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
scope "/", HelloWeb do
|
||||
pipe_through :browser
|
||||
|
||||
resources "/posts", PostController, only: [:index, :show]
|
||||
|
||||
get "/", PageController, :home
|
||||
end
|
||||
```
|
||||
|
||||
Let's create a couple of documents to see how our app renders them.
|
||||
|
||||
```bash
|
||||
$ touch documents/{hello-there.md,welcome-to-our-demo-app.md}
|
||||
```
|
||||
|
||||
And now we visit `localhost:4000/posts`.
|
||||
|
||||
|
||||
|
||||
Let's add some markdown content to `documents/hello-there.md` so we can see how
|
||||
the `show` template looks.
|
||||
|
||||
::: filename-for-code-block
|
||||
`documents/hello-there.md`
|
||||
:::
|
||||
|
||||
```markdown
|
||||
# hello
|
||||
|
||||
This is a paragraph.
|
||||
|
||||
This is a code block.
|
||||
|
||||
> This is a block quote.
|
||||
|
||||
## this is a heading
|
||||
|
||||
- this is
|
||||
- a list
|
||||
- of items
|
||||
```
|
||||
|
||||
When we visit `localhost:4000/posts/hello-there`, it looks like this:
|
||||
|
||||
|
||||
|
||||
Our document's content is visible, but it's missing any styling. We will fix
|
||||
this by adding the `typography` plugin to Tailwind's config file.
|
||||
|
||||
## 5. Style the markdown content
|
||||
|
||||
Let's add `@tailwindcss/typography` to the plugins list in
|
||||
`tailwind.config.js`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`assets/tailwind.config.js`
|
||||
:::
|
||||
|
||||
```javascript
|
||||
plugins: [
|
||||
// ...
|
||||
require("@tailwindcss/typography")
|
||||
]
|
||||
```
|
||||
|
||||
Then we'll add Tailwind's `prose` class to our post's HTML content.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello_web/controllers/post_html/show.html.heex`
|
||||
:::
|
||||
|
||||
```heex
|
||||
<div class="prose">
|
||||
<%= raw(@post.body) %>
|
||||
</div>
|
||||
```
|
||||
|
||||
After restarting our app, we can visit the post `show` page again, and this
|
||||
time our content is styled appropriately.
|
||||
|
||||
|
||||
|
||||
## 6. Auto-reload the browser when markdown content changes
|
||||
|
||||
Since our `pandoc` file-watcher converts documents to HTML whenever changes are
|
||||
detected, all we need to do to have the changes update in our browser in
|
||||
real-time is add the static files path to the `live_reload` config in
|
||||
`config/dev.exs`.
|
||||
|
||||
::: warning
|
||||
**Caveat: Long Documents**
|
||||
|
||||
If the documents we are editing become long, there are two issues that may
|
||||
arise.
|
||||
|
||||
1. If the page reloads, but our document content is missing, that means the
|
||||
live-reloader reloaded the page before the document was finished being
|
||||
converted. To address this, we can use the `interval` option to set a lengh
|
||||
of time greater than the `100` ms default value.
|
||||
|
||||
2. If our terminal is being flooded with too many `[debug] Live reload...`
|
||||
messages, we can use the `debounce` option to set a delay before a reload
|
||||
event is sent to the browser.
|
||||
:::
|
||||
|
||||
::: filename-for-code-block
|
||||
`config/dev.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
config :hello, HelloWeb.Endpoint,
|
||||
live_reload: [
|
||||
interval: 1000,
|
||||
debounce: 200,
|
||||
patterns: [
|
||||
# ...
|
||||
~r"priv/static/posts/.*(html)$"
|
||||
]
|
||||
]
|
||||
```
|
||||
|
||||
Now we can edit the markdown in our documents and, as soon as we save our
|
||||
changes, the new content should appear in our browser.
|
||||
|
||||
## 7. Handle draft documents
|
||||
|
||||
It would be very helpful if we had a place to keep draft documents that are
|
||||
visible to us during development but absent in a production release. With a few
|
||||
changes to `lib/hello/documents.ex` we can do just that.
|
||||
|
||||
First, let's make a directory for draft documents.
|
||||
|
||||
```bash
|
||||
$ mkdir documents/_drafts
|
||||
```
|
||||
|
||||
So our published documents will live in the top-level `documents` directory,
|
||||
and our draft documents will live in the subdirectory `documents/_drafts`.
|
||||
|
||||
The first change to the `list_posts/1` function in `documents.ex` adds a
|
||||
conditional depending on `Mix.env()` for what directories to scan. In
|
||||
production it will only scan `documents/*` for files, while in development it
|
||||
will scan all subdirectories with `documents/**/*`.
|
||||
|
||||
The second change required to `list_posts/1` is to filter out directories,
|
||||
since `Path.wildcard/1` will include the directory `_drafts` in with the list
|
||||
with files.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello/documents.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
def list_posts do
|
||||
"documents"
|
||||
|> Path.join(if(Mix.env() != :prod, do: "**/*", else: "*"))
|
||||
|> Path.wildcard()
|
||||
|> Enum.filter(&(File.stat!(&1).type != :directory))
|
||||
|> Enum.map(fn path ->
|
||||
# ...
|
||||
```
|
||||
|
||||
Now we just need to update the `get_post!/1` function. In order to make sure
|
||||
that we never accidentally publish draft documents, we will convert them on the
|
||||
fly as necessary and will store their content in memory rather that writing to
|
||||
disk. Since the `pandoc` profile `default` outputs conversion results
|
||||
to `stdout`, we can use that profile instead of `hello` when running the
|
||||
conversion and simply capture the results with `ExUnit.CaptureIO.capture_io/1`.
|
||||
|
||||
::: filename-for-code-block
|
||||
`lib/hello/documents.ex`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
def get_post!(id) do
|
||||
# ...
|
||||
body =
|
||||
if "_drafts" in Path.split(post.path) do
|
||||
ExUnit.CaptureIO.capture_io(fn ->
|
||||
Mix.Task.rerun("pandoc", ["default", post.path])
|
||||
end)
|
||||
else
|
||||
:hello
|
||||
|> :code.priv_dir()
|
||||
|> Path.join("static/posts/#{id}.html")
|
||||
|> File.read!()
|
||||
end
|
||||
# ...
|
||||
end
|
||||
```
|
||||
|
||||
Lastly, since we are not writing draft documents to disk, we need to add
|
||||
another pattern for Phoenix's LiveReloader to reload the browser when draft
|
||||
content changes.
|
||||
|
||||
::: filename-for-code-block
|
||||
`config/dev.exs`
|
||||
:::
|
||||
|
||||
```elixir
|
||||
config :hello, HelloWeb.Endpoint,
|
||||
live_reload: [
|
||||
patterns: [
|
||||
# ...
|
||||
~r"priv/static/posts/.*(html)$",
|
||||
~r"documents/_drafts/.*(md)$"
|
||||
]
|
||||
]
|
||||
```
|
||||
|
||||
## 8. Release
|
||||
|
||||
When we next release our application, we simply need to run `mix
|
||||
statics.deploy` to build assets and documents first, and then run the release
|
||||
command.
|
||||
|
||||
```bash
|
||||
$ mix statics.deploy
|
||||
$ MIX_ENV=prod mix release
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
|
||||
That's it! This solution allows us to write our posts in simple markdown and
|
||||
see them converted to HTML automatically with Phoenix and Pandoc. Furthermore,
|
||||
we are able to use Phoenix's powerful template language HEEx to make writing
|
||||
HTML faster and easier, so we can focus more on content and less on
|
||||
development.
|
||||
|
||||
We hope this post was as useful to others as it has been to us. We are always
|
||||
appreciative of any
|
||||
[feedback](mailto:webdevcat@proton.me?subject=re:%20post%20publish-markdown-documents-as-static-web-pages-with-pandoc-and-phoenix)
|
||||
readers would like to share with us. Thanks for reading and happy coding!
|
||||
Reference in New Issue
Block a user