doc: comline doc, the package's documentation

Cargo.toml

@@ -32,15 +32,18 @@ eyre = "0.6.8"
 # needs real I/O (git fetch, filesystem), so it's off by default for
 # `comline-core`'s wasm32 consumers (the playground editor, the language
 # server); the CLI is the one consumer that actually runs `comline build`.
-comline-core = { git = "https://github.com/ComlineProject/core", rev = "47066583002dbca27975827106f9e34761d0dc64", features = ["deps"] }
+comline-core = { git = "https://github.com/ComlineProject/core", rev = "076cae0c3a0639b0c083b61a94f9d4913e853b3f", features = ["deps"] }
 # All three pinned to revs that agree on one `comline-codegen`, and bumped
 # together: comline-rust and comline-typescript's codegen crates consume
 # `comline-codegen`'s types, so the rev below must be byte-identical to the
 # one *they* pin - Cargo only unifies a git dependency across identical rev
 # strings, and two copies of `GenRequest` are two incompatible types.
-comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "0601f004302ebb93e6804732419ccf459902e84a" }
-comline-codegen-rust = { git = "https://github.com/ComlineProject/comline-rust", rev = "6e8715ac126ab78316dd6a2712165d716351fab1", optional = true }
-comline-codegen-typescript = { git = "https://github.com/ComlineProject/comline-typescript", rev = "470dc5bad2f7e36e1cc2f44e3e57fde864a42315", optional = true }
+comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "7b51264b216ef4c5e2afbe36897a5b35dafe7e06" }
+# The official Markdown documentation renderer (`comline doc`), linked in
+# like the official generators. Same generation rev as `comline-codegen`.
+comline-doc-markdown = { git = "https://github.com/ComlineProject/generation", rev = "7b51264b216ef4c5e2afbe36897a5b35dafe7e06" }
+comline-codegen-rust = { git = "https://github.com/ComlineProject/comline-rust", rev = "bef9a74c028d7a44b4dc13ed677d300e061b2147", optional = true }
+comline-codegen-typescript = { git = "https://github.com/ComlineProject/comline-typescript", rev = "47b54cc75b05bc52586d431d26ed4def316cf0c4", optional = true }
 # `comline conformance`: the conformance runner (corpus loading, judging,
 # profile proofs) and its case format, a public spec. Pinned by git rev.
 comline-runtime-conformance = { git = "https://github.com/ComlineProject/runtime", rev = "6f8f73a247f0aa4ac8b7041b2fdc4b02bce0e0d6" }

docs/cli.md

@@ -20,6 +20,7 @@ This is the long-form reference. For a two-minute tour see the
   - [`build`](#comline-build)
   - [`generate`](#comline-generate)
   - [`diff`](#comline-diff)
+  - [`doc`](#comline-doc)
   - [`conformance`](#comline-conformance)
   - [`clean`](#comline-clean)
   - [`reset`](#comline-reset)
@@ -445,6 +446,60 @@ comline diff 0.0.1 HEAD
 Fails with exit `2` if the project has never been built, and exit `1` if an
 argument matches no stored version (the error lists the versions that exist).
 
+### `comline doc`
+
+```
+comline doc [--out <dir>] [--version <v>] [--changelog-from <v>] [--format markdown|json]
+```
+
+Write the package's documentation. Nothing is written to `.comline/`.
+
+- **Pages** (`markdown`, the default, rendered by the official renderer linked
+  into `comline`): `index.md` (the package, its `//!` header, its modules), one
+  page per module (`types::user` is `types/user.md`) with its `//!` header and
+  every unit with its `///` docs, fields, arguments and each protocol's wire
+  summary, `changelog.md` with `--changelog-from`, and `nav.yml`, a `nav` for
+  mkdocs or zensical.
+- **`json`** writes the documentation model itself, `model.json`, in the shape
+  of core's `schemas/doc-model.json`.
+
+| Flag | Default | Meaning |
+|---|---|---|
+| `--out <dir>` | `[doc] out`, else `docs/api` | where to write, relative to the project |
+| `--version <v>` | the working tree | document a built version (a version string, a commit hash, or `HEAD`) from its frozen IR, `//!` headers included |
+| `--changelog-from <v>` | `[doc] changelog_from` | add what changed from built version `<v>` to the documented one |
+| `--format <f>` | `[doc] format`, else `markdown` | `markdown`, `json`, or another id the `[doc] renderer` offers |
+
+A built version keeps no source, so with `--version` its protocols have no wire
+summary. A version built before headers were stored has none.
+
+`comline.toml`'s `[doc]` table:
+
+```toml
+[doc]
+out = "docs/api"
+format = "markdown"
+renderer = "renderers/my-renderer.wasm"  # optional: an external renderer
+timeout = 60                             # seconds per call to `renderer`
+links = { std = "https://comlineproject.github.io/docs/std/" }
+changelog_from = "0.1.0"
+```
+
+- `renderer` is run exactly like an external `generator`: a `.wasm` under
+  wasmtime with no filesystem, environment or network, a native executable with
+  an empty environment, the same limits and path checks. It must list the
+  format in its `Describe` answer's `renderers`.
+- `links` maps a dependency (or `std`) to the base URL of its documentation. A
+  type from it links to `<base>/<module>/#<anchor>`; a dependency not listed is
+  plain text.
+
+```bash
+comline doc
+comline doc --out site/api --changelog-from 0.1.0
+comline doc --version 0.2.0
+comline doc --format json
+```
+
 ### `comline conformance`
 
 ```

src/cli.rs

@@ -100,6 +100,47 @@ Examples:
         json: bool,
     },
 
+    /// Write the package's documentation: Markdown pages, or the model as JSON
+    #[command(long_about = "\
+Write the package's documentation: an index page, one page per module (its \
+`//!` header, every unit with its `///` docs, fields, arguments and each \
+protocol's wire summary), a changelog page with `--changelog-from`, and an \
+mkdocs/zensical `nav.yml`. Read-only: nothing is written to `.comline/`.
+
+The working tree is documented unless `--version` names a built version (a \
+version string, a commit hash or `HEAD`), which is documented from its frozen \
+IR, `//!` headers included. A built version keeps no source, so its protocols \
+have no wire summary.
+
+`--format json` writes the documentation model itself (`model.json`), for \
+other tools. `comline.toml`'s `[doc]` table sets the defaults: `out` \
+(`docs/api`), `format` (`markdown`), `renderer` (an external renderer, run \
+like an external generator), `links` (dependency docs URLs) and \
+`changelog_from`.
+
+Examples:
+  comline doc
+  comline doc --out site/api --changelog-from 0.1.0
+  comline doc --version 0.2.0
+  comline doc --format json")]
+    Doc {
+        /// Output directory; default from `[doc] out`, else `docs/api`
+        #[arg(long, value_name = "DIR")]
+        out: Option<std::path::PathBuf>,
+
+        /// Document this built version instead of the working tree
+        #[arg(long = "version", value_name = "VERSION")]
+        doc_version: Option<String>,
+
+        /// Add a changelog from this built version to the documented one
+        #[arg(long, value_name = "VERSION")]
+        changelog_from: Option<String>,
+
+        /// `markdown` (pages) or `json` (the model); default from `[doc] format`
+        #[arg(long, value_name = "FORMAT")]
+        format: Option<String>,
+    },
+
     /// Validate the project and write generated code for each configured target
     #[command(long_about = "\
 Validate the project (no version bump, no `.comline/` write), then write \

src/commands/doc.rs

@@ -0,0 +1,173 @@
+//! `comline doc` — the package's documentation (core#145).
+//!
+//! Core builds the documentation model (`comline_core::doc`): from the
+//! compiled working tree, or from a built version's frozen IR and `//!`
+//! headers with `--version`. `--changelog-from` adds what changed since an
+//! earlier built version. `--format json` writes the model itself; any other
+//! format is a renderer's id, `markdown` being the official one, linked in
+//! like the official generators. `[doc] renderer` in `comline.toml` names an
+//! external renderer instead, run exactly like an external generator
+//! ([`crate::external`]: the same sandbox, limits and path checks).
+//!
+//! Like `check`, nothing is written to `.comline/`.
+
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use comline_codegen::config::{ComlineToml, DEFAULT_DOC_FORMAT, DEFAULT_DOC_OUT};
+use comline_codegen::protocol::{FileOut, RenderOptions, RenderRequest};
+use comline_codegen::Registry;
+use comline_core::doc::{self, DocModel, DOC_MODEL_VERSION};
+use comline_core::package::build::cas::objects::Tree;
+use comline_core::package::build::cas::ObjectStore;
+use comline_core::schema::ir::frozen::unit::FrozenUnit;
+use miette::{miette, IntoDiagnostic, Result, WrapErr};
+
+use crate::commands::{core_err, ensure_project};
+use crate::{external, history, ui};
+
+/// The command line's say, each overriding its `[doc]` key.
+#[derive(Debug, Default)]
+pub struct Flags {
+    pub out: Option<PathBuf>,
+    pub version: Option<String>,
+    pub changelog_from: Option<String>,
+    pub format: Option<String>,
+}
+
+/// The renderers linked into this binary.
+pub(crate) fn renderer_registry() -> Registry {
+    let mut registry = Registry::new();
+    comline_doc_markdown::register(&mut registry);
+    registry
+}
+
+pub fn run(work_dir: &Path, flags: Flags) -> Result<()> {
+    ensure_project(work_dir)?;
+    let cfg = ComlineToml::load(work_dir)?.doc;
+
+    let out = match (&flags.out, &cfg.out) {
+        (Some(out), _) => work_dir.join(out),
+        (None, Some(out)) => work_dir.join(out),
+        (None, None) => work_dir.join(DEFAULT_DOC_OUT),
+    };
+    let format = flags
+        .format
+        .or(cfg.format.clone())
+        .unwrap_or_else(|| DEFAULT_DOC_FORMAT.to_string());
+    let changelog_from = flags.changelog_from.or(cfg.changelog_from.clone());
+
+    let (mut model, schemas) = match &flags.version {
+        Some(spec) => frozen_model(work_dir, spec)?,
+        None => working_tree_model(work_dir)?,
+    };
+
+    if let Some(from) = &changelog_from {
+        let history = history::load(work_dir)?;
+        let commit = history::resolve(&history, from)?;
+        let store = ObjectStore::new(work_dir);
+        let old = history::load_schemas(&store, commit)?;
+        let to = model
+            .package
+            .version
+            .clone()
+            .unwrap_or_else(|| "working tree".to_string());
+        model
+            .changelog
+            .push(doc::changelog(&commit.version, &old, &to, &schemas));
+    }
+
+    let files = if format == "json" {
+        let json = serde_json::to_string_pretty(&model).into_diagnostic()?;
+        vec![FileOut {
+            path: "model.json".into(),
+            contents: format!("{json}\n").into_bytes(),
+        }]
+    } else {
+        let request = RenderRequest {
+            renderer: format.clone(),
+            model_version: DOC_MODEL_VERSION,
+            model: serde_json::to_string(&model).into_diagnostic()?,
+            options: RenderOptions {
+                links: cfg.links.clone(),
+            },
+        };
+        match &cfg.renderer {
+            Some(path) => {
+                let timeout = cfg
+                    .timeout
+                    .map(Duration::from_secs)
+                    .unwrap_or(external::DEFAULT_TIMEOUT);
+                let renderer = external::External::open(&work_dir.join(path), timeout)?;
+                let description = renderer.describe()?;
+                renderer.render(&description, request)?
+            }
+            None => external::builtin_render(&renderer_registry(), request)?,
+        }
+    };
+
+    for file in &files {
+        let path = out.join(file.path_buf());
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)
+                .into_diagnostic()
+                .wrap_err_with(|| format!("failed to create `{}`", parent.display()))?;
+        }
+        std::fs::write(&path, &file.contents)
+            .into_diagnostic()
+            .wrap_err_with(|| format!("failed to write `{}`", path.display()))?;
+    }
+    ui::success(format!(
+        "Documented {} into {} ({} file(s))",
+        match &model.package.version {
+            Some(v) => format!("{} {v}", model.package.name),
+            None => model.package.name.clone(),
+        },
+        out.display(),
+        files.len()
+    ));
+    Ok(())
+}
+
+/// The working tree's model, and its frozen schemas for a changelog.
+fn working_tree_model(work_dir: &Path) -> Result<(DocModel, Vec<Vec<FrozenUnit>>)> {
+    let spinner = ui::spinner("compiling schemas");
+    let outcome = comline_core::package::build::compile_package(work_dir);
+    ui::finish_spinner(spinner);
+    let context = outcome.map_err(|e| core_err("validation failed", e))?;
+    Ok((doc::model(&context, None), doc::frozen_schemas(&context)))
+}
+
+/// A built version's model, from its frozen IR and headers.
+fn frozen_model(work_dir: &Path, spec: &str) -> Result<(DocModel, Vec<Vec<FrozenUnit>>)> {
+    let history = history::load(work_dir)?;
+    let commit = history::resolve(&history, spec)?;
+    let store = ObjectStore::new(work_dir);
+    let schemas = history::load_schemas(&store, commit)?;
+    let config = history::load_config(&store, commit)?;
+    // A built version's config doesn't always freeze the package's name;
+    // the working tree's is the same unless the package was renamed.
+    let name = comline_core::package::config::ir::frozen::namespace(&config)
+        .map(str::to_string)
+        .or_else(|| {
+            comline_core::package::build::compile_package_without_fetching(work_dir)
+                .ok()
+                .map(|context| context.config.name.value.to_string())
+        })
+        .or_else(|| {
+            work_dir
+                .canonicalize()
+                .ok()?
+                .file_name()
+                .map(|n| n.to_string_lossy().into_owned())
+        })
+        .ok_or_else(|| miette!("version `{spec}` has no package name"))?;
+    let root = store
+        .read(&commit.tree)
+        .and_then(|bytes| Tree::from_bytes(&bytes))
+        .map_err(|e| miette!("failed to read version `{spec}`: {e:#}"))?;
+    let headers = doc::read_docs(&store, &root)
+        .map_err(|e| miette!("failed to read version `{spec}`'s headers: {e:#}"))?;
+    let model = doc::model_frozen(&name, Some(commit.version.clone()), &schemas, headers);
+    Ok((model, schemas))
+}

src/commands/mod.rs

@@ -9,6 +9,7 @@ pub mod clean;
 pub mod completions;
 pub mod conformance;
 pub mod diff;
+pub mod doc;
 pub mod generate;
 pub mod inspect;
 pub mod new;