diff --git a/conftest.py b/conftest.py index 5ae8166f..3a8e7bf5 100644 --- a/conftest.py +++ b/conftest.py @@ -77,6 +77,11 @@ def _disable_telemetry(monkeypatch): """ monkeypatch.setenv("DEEPCTL_TELEMETRY_DISABLED", "1") monkeypatch.setenv("DEEPCTL_TELEMETRY_DSN", "http://k@127.0.0.1:9/1") + # A developer's DO_NOT_TRACK or DEEPCTL_TELEMETRY=log must not change + # what telemetry's own tests see. (CI is cleared only by those tests: + # other suites read it as an automation signal.) + monkeypatch.delenv("DO_NOT_TRACK", raising=False) + monkeypatch.delenv("DEEPCTL_TELEMETRY", raising=False) @pytest.fixture(autouse=True) diff --git a/docs/contracts/help-cache.md b/docs/contracts/help-cache.md index 830a8a8c..c2f67611 100644 --- a/docs/contracts/help-cache.md +++ b/docs/contracts/help-cache.md @@ -77,7 +77,7 @@ The launcher serves an entry only when every one of the following holds. Anythin 4. **Each name in `env`** has the same value in the launcher's environment, and `null` means the variable must be unset. On Windows, look the names up case-insensitively, as the OS does. A variable that isn't listed doesn't matter. `DEEPCTL_PROG_NAME` and `DEEPCTL_HELP_CACHE` are never listed: the launcher sets them, and `prog` already records the name. 5. **`text_sha256`** (lowercase hex) equals the SHA-256 of the entry bytes the launcher read. This binds the text to its key: Python writes the text first and the key second, so text from another run under this key is a miss and is never served. -A bundle install (`-I`, no user site, `HOME` is the real home) lists about 10 files and 18 environment variables. Python writes no key over the limits (32 files, 64 variables), so an install with a long `sys.path` (an editable install, say) gets no entry and always takes the Python path. +A bundle install (`-I`, no user site, `HOME` is the real home) lists about 10 files and 21 environment variables. Python writes no key over the limits (32 files, 64 variables), so an install with a long `sys.path` (an editable install, say) gets no entry and always takes the Python path. **Paths and values that aren't valid Unicode.** - **Unix:** a path or environment value with bytes that don't decode under the filesystem encoding reaches Python as a lone surrogate (`\udcXX`). diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 00000000..a9ad2f1f --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,64 @@ +# Telemetry + +dg sends one error report when a command fails with an unexpected error. Nothing is sent when a command succeeds, for a usage error or another error dg reports to you, or for `--help` and `--version`. The README's Telemetry section says where reports go. + +## When it runs + +- **At most one report per run**, and only for an unexpected error. +- **No network or reporting code on the success path.** The reporting library is loaded only after an unexpected error, so a command that succeeds opens no connection for telemetry. +- **A short time budget.** dg waits about 0.6 s at most for a report, loading included, and doesn't wait at all when no report was queued. With blocked or slow egress, a failing command takes at most that much longer. + +## Turn it off + +Any one of these turns telemetry off. They are checked in this order: + +| Switch | Scope | +|---|---| +| `--no-telemetry` | One command | +| `DEEPCTL_TELEMETRY_DISABLED=1` | A shell session, a CI job | +| `DEEPCTL_TELEMETRY=off` | A shell session, a CI job | +| `DO_NOT_TRACK=1` | Every tool that follows the [Console Do Not Track](https://consoledonottrack.com/) convention | +| `CI` set (as CI services do) | Any CI job | +| `telemetry.enabled: false` in your `config.yaml` | Always | + +For `DEEPCTL_TELEMETRY_DISABLED`, `DO_NOT_TRACK` and `CI`, any value except an empty one, `0`, `false`, `no` or `off` turns telemetry off. `DEEPCTL_TELEMETRY` turns it off when set to `off`, `0`, `false`, `no` or `disabled`; `log` is described below, and any other value changes nothing. A config file dg won't load (unreadable, too large, not valid YAML, a repeated `telemetry` key, or one dg doesn't trust in an AI agent session) counts as turned off too. A project's `deepgram.yaml` can't change these settings. The last line of `dg --help` shows whether telemetry is on. + +## See what would be sent + +Run a command with `DEEPCTL_TELEMETRY=log`. If it fails with an unexpected error and telemetry is on, dg prints the report it would have sent to stderr, as one JSON line, and sends nothing. + +```sh +DEEPCTL_TELEMETRY=log dg listen recording.wav +``` + +## Field schema + +A report holds these fields and nothing else. A value that doesn't match its expected shape (an identifier, a version string, a command path) is dropped, never shortened or escaped. Names from code outside dg and the libraries it calls, such as a plugin's, are sent as ``. + + +| Field | Contents | +|---|---| +| `event_id` | Random report ID (32 hex characters) | +| `timestamp` | When the error happened | +| `platform` | Always `python` | +| `level` | Always `error` | +| `release` | `deepctl@` and the dg version | +| `environment` | Always `production` | +| `sdk.name` | Name of the reporting library | +| `sdk.version` | Version of the reporting library | +| `tags.cmd.path` | Command path, such as `dg listen` | +| `tags.cmd.flags` | Names of the flags used, never their values, such as `--model,--output` | +| `tags.cli.os` | Operating system, such as `Darwin` | +| `tags.cli.arch` | CPU architecture, such as `arm64` | +| `tags.cli.python` | Python version, such as `3.12.4` | +| `exception.values[].type` | Exception type, and the type of each exception that led to it | +| `exception.values[].mechanism.type` | How the error was caught, such as `generic` | +| `exception.values[].mechanism.handled` | Whether dg caught it | +| `exception.values[].stacktrace.frames[].module` | Module of each stack frame | +| `exception.values[].stacktrace.frames[].function` | Function of each stack frame | +| `exception.values[].stacktrace.frames[].lineno` | Line number of each stack frame | + + +A report never holds flag values or arguments, file names or paths, environment variables, API keys, request or response data, error messages, log output, variable values, source code, your hostname or your username. + +A test (`packages/deepctl-telemetry/tests/unit/test_contract.py`) sends a report through the real error path and checks that its fields are exactly the ones in this table. diff --git a/packages/deepctl-cmd-init/src/deepctl_cmd_init/clones.py b/packages/deepctl-cmd-init/src/deepctl_cmd_init/clones.py new file mode 100644 index 00000000..6beae8b5 --- /dev/null +++ b/packages/deepctl-cmd-init/src/deepctl_cmd_init/clones.py @@ -0,0 +1,192 @@ +"""dg init's record of the clones it made, in dg's own state directory. + +A rerun of ``dg init`` resumes in a directory only when this record names +it: nothing inside the directory (which anyone could have packed into an +archive) is trusted for that. Each record is one owner-only file under +``{state_dir}/init-clones/`` (0700), named after the directory's real path, +holding that path, the template, its repository, the commit dg cloned and +the phases done, and the clone directory's device and inode, so a +directory deleted and made again at the same path is not taken for it. +""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import re +import secrets +import stat +import sys +from dataclasses import asdict, dataclass +from typing import Any + +from deepctl_core.login_state import StateDirError, _open_dir, state_dir + +DIR_NAME = "init-clones" +_MAX_BYTES = 16 * 1024 +_COMMIT = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") +_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) +_NONBLOCK = getattr(os, "O_NONBLOCK", 0) +_DIRECTORY = getattr(os, "O_DIRECTORY", 0) +_BINARY = getattr(os, "O_BINARY", 0) + + +@dataclass(frozen=True) +class CloneRecord: + path: str + template: str + url: str + commit: str + phases: list[str] + dev: int + ino: int + + +def _name(real_path: str) -> str: + return hashlib.sha256(real_path.encode("utf-8", "surrogateescape")).hexdigest() + + +def _use_dir_fd() -> bool: + return sys.platform != "win32" and os.open in os.supports_dir_fd + + +def _open_records(*, create: bool) -> int: + """Open the records directory inside the state directory, both checked: + real directories owned by this user, the records directory 0700.""" + parent = _open_dir(state_dir(), create=create) + try: + if create: + try: + if _use_dir_fd(): + os.mkdir(DIR_NAME, 0o700, dir_fd=parent) + else: + os.mkdir(state_dir() / DIR_NAME, 0o700) + except FileExistsError: + pass + flags = os.O_RDONLY | _DIRECTORY | _NOFOLLOW + if _use_dir_fd(): + fd = os.open(DIR_NAME, flags, dir_fd=parent) + else: + fd = os.open(state_dir() / DIR_NAME, flags) + finally: + os.close(parent) + try: + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode): + raise StateDirError(errno.ENOTDIR, "not a directory", DIR_NAME) + if sys.platform != "win32": + if st.st_uid != os.geteuid(): + raise StateDirError(errno.EPERM, "owned by another user", DIR_NAME) + if stat.S_IMODE(st.st_mode) & 0o077: + if not create: + raise StateDirError(errno.EPERM, "not private", DIR_NAME) + os.fchmod(fd, 0o700) + except BaseException: + os.close(fd) + raise + return fd + + +def _valid(data: Any, real_path: str) -> CloneRecord | None: + if not isinstance(data, dict) or data.get("path") != real_path: + return None + template, url, commit = data.get("template"), data.get("url"), data.get("commit") + phases = data.get("phases") + if not (isinstance(template, str) and isinstance(url, str)): + return None + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + return None + if not isinstance(phases, list) or not all(isinstance(p, str) for p in phases): + return None + dev, ino = data.get("dev"), data.get("ino") + if not isinstance(dev, int) or isinstance(dev, bool): + return None + if not isinstance(ino, int) or isinstance(ino, bool): + return None + return CloneRecord(real_path, template, url, commit, list(phases), dev, ino) + + +def load(real_path: str) -> CloneRecord | None: + """The record for ``real_path``, or None when there is no usable one. + + Only an owner-only regular file with one link, opened without following + a link or blocking on a FIFO, in the checked records directory. + """ + try: + dir_fd = _open_records(create=False) + except OSError: + return None + name = _name(real_path) + ".json" + try: + flags = os.O_RDONLY | _NOFOLLOW | _NONBLOCK | _BINARY + if _use_dir_fd(): + fd = os.open(name, flags, dir_fd=dir_fd) + else: + fd = os.open(state_dir() / DIR_NAME / name, flags) + except OSError: + os.close(dir_fd) + return None + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode) or st.st_size > _MAX_BYTES: + return None + if sys.platform != "win32" and ( + st.st_uid != os.geteuid() + or stat.S_IMODE(st.st_mode) & 0o077 + or st.st_nlink != 1 + ): + return None + raw = os.read(fd, _MAX_BYTES + 1) + except OSError: + return None + finally: + os.close(fd) + os.close(dir_fd) + try: + return _valid(json.loads(raw.decode("utf-8")), real_path) + except ValueError: + return None + + +def save(record: CloneRecord) -> None: + """Write ``record`` atomically: a new 0600 file (``O_CREAT|O_EXCL| + O_NOFOLLOW``) next to the target, then renamed over it. Raises OSError + (StateDirError for a state directory dg can't use).""" + data = (json.dumps(asdict(record), indent=2) + "\n").encode("utf-8") + name = _name(record.path) + ".json" + tmp = f".{name}.{secrets.token_hex(8)}.tmp" + dir_fd = _open_records(create=True) + base = state_dir() / DIR_NAME + try: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW | _BINARY + if _use_dir_fd(): + fd = os.open(tmp, flags, 0o600, dir_fd=dir_fd) + else: + fd = os.open(base / tmp, flags, 0o600) + try: + try: + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + os.fsync(fd) + finally: + os.close(fd) + if _use_dir_fd(): + os.replace(tmp, name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) + else: + os.replace(base / tmp, base / name) + except BaseException: + try: + if _use_dir_fd(): + os.unlink(tmp, dir_fd=dir_fd) + else: + os.unlink(base / tmp) + except OSError: + pass + raise + finally: + os.close(dir_fd) diff --git a/packages/deepctl-cmd-init/src/deepctl_cmd_init/command.py b/packages/deepctl-cmd-init/src/deepctl_cmd_init/command.py index 8da8359a..e8343afb 100644 --- a/packages/deepctl-cmd-init/src/deepctl_cmd_init/command.py +++ b/packages/deepctl-cmd-init/src/deepctl_cmd_init/command.py @@ -2,6 +2,9 @@ from __future__ import annotations +import os +import shlex +import sys from pathlib import Path from typing import Any @@ -12,12 +15,94 @@ Config, DeepgramClient, ) -from deepctl_core.output import _agentic +from deepctl_core import messages as core_msg +from deepctl_core.output import _agentic, is_ai_agent, print_error, print_hint from rich.console import Console +from rich.markup import escape from rich.table import Table -from . import lifecycle, templates_api -from .models import InitResult, TemplateDetail, TemplateInfo +from . import clones, lifecycle, templates_api +from . import messages as msg +from .models import ( + InitCodedError, + InitResult, + LifecycleStep, + TemplateDetail, + TemplateInfo, +) + +_MAKE_STEPS = ["make check-prereqs", "make init"] + + +def _hard_signal() -> bool: + """An AI agent marked the environment, or ``DG_AGENT=1`` says one runs dg.""" + return is_ai_agent() or os.environ.get("DG_AGENT") == "1" + + +def _person_at_terminal() -> bool: + """A person can answer here: no agent or automation signal, and stdin + and stderr are terminals. Otherwise setup needs --yes and the start + step --start.""" + if _agentic or _hard_signal(): + return False + try: + return sys.stdin.isatty() and sys.stderr.isatty() + except (AttributeError, ValueError): + return False + + +def _commands(step: LifecycleStep | None) -> list[str]: + if step is None or step.command is None: + return [] + return step.command if isinstance(step.command, list) else [step.command] + + +def _rerun(template: str, dest: Path, *flags: str) -> str | None: + """``{prog} init {template} --dir {dest} {flags}``, or None when it + can't be given as one runnable command.""" + from deepctl_core.transport import prog + + program = prog() + if program is None: + return None + tokens = ["init", template, "--dir", str(dest), *flags] + if any(token.startswith("=") for token in tokens): + return None # zsh's EQUALS option would expand it + if os.name == "nt" and any(shlex.quote(token) != token for token in tokens): + return None + step = f"{program} {shlex.join(tokens)}" + return step if shlex.join(shlex.split(step)) == step else None + + +def _coded( + error: str, + code: str, + hint: str, + next_step: str | None, + details: dict[str, Any] | None = None, +) -> InitCodedError: + """Report a coded error on stderr and return its payload.""" + print_error(escape(error)) + print_hint(escape(hint)) + return InitCodedError( + error=error, code=code, hint=hint, next_step=next_step, details=details + ) + + +def _prog_text() -> str: + """``{prog}`` in a message: the running command, else ``dg``.""" + from deepctl_core.transport import prog + + return prog() or "dg" + + +def _still_same(dest: Path, identity: tuple[int, int] | None) -> bool: + """Whether ``dest`` is still the directory dg cloned into or resumed + in: checked again right before the API key is written there and before + any setup or start command runs, so a directory swapped in meanwhile + is refused.""" + return identity is not None and lifecycle.same_directory(dest, *identity) + console = Console() status_console = Console(stderr=True) @@ -46,6 +131,7 @@ class InitCommand(BaseCommand): "dg init node-transcription --dir ./my-app", "dg init node-transcription --install", "dg init node-transcription --no-install", + "dg init node-transcription --yes", "dg init --list", "dg init --list --search flask", ] @@ -53,8 +139,10 @@ class InitCommand(BaseCommand): "Scaffold a Deepgram starter app from the templates gallery. " "Pass the template name/slug directly (e.g. 'node-transcription'). " "Use --list to enumerate available templates. " - "After cloning, runs make check-prereqs then make init automatically. " - "Use --no-install to skip setup. " + "Setup commands (make check-prereqs, make init, or the template's own) " + "run only with --yes: show them to the user and pass --yes only if the " + "user approves, or pass --no-install to skip them. The app is started " + "only with --start. A rerun into the same --dir resumes there. " "Requires git, node, npm, make, and curl to be installed." ) @@ -88,7 +176,10 @@ def get_arguments(self) -> list[dict[str, Any]]: }, { "names": ["--install"], - "help": "Run setup (make check-prereqs && make init) without prompting", + "help": ( + "Run the setup commands without asking (in an AI agent " + "session, use --yes)" + ), "is_flag": True, "is_option": True, }, @@ -110,6 +201,15 @@ def get_arguments(self) -> list[dict[str, Any]]: "is_flag": True, "is_option": True, }, + { + "names": ["--yes", "-y"], + "help": ( + "Approve and run the template's setup commands without " + "asking; the only way in an AI agent session" + ), + "is_flag": True, + "is_option": True, + }, ] def handle( @@ -128,6 +228,7 @@ def handle( no_install: bool = kwargs.get("no_install", False) do_start: bool = kwargs.get("start", False) no_start: bool = kwargs.get("no_start", False) + yes: bool = kwargs.get("yes", False) # ── List mode ────────────────────────────────────────────────────────── if list_mode: @@ -159,51 +260,104 @@ def handle( status="error", message=f"Template not found: {template_slug}" ) elif _agentic: - # In agent mode, require an explicit template name - return InitResult( - status="error", - message=( - "In agent mode, pass the template slug directly, e.g.: " - "dg init node-transcription\n" - "Use dg init --list to see available templates." - ), - ) + # No picker in agent mode: the template is named on the command. + return self._pick_needs_name() else: detail = self._interactive_pick(search) if not detail: return InitResult(status="cancelled", message="No template selected") + # ── Check what the template service sent ─────────────────────────────── + # The repository and the template name come from the service; both + # are checked before either is used in a path, a command or output. + github_url = detail.links.get("github") + if not github_url: + return InitResult(status="error", message="Template has no GitHub URL") + if not lifecycle.is_template_repo(github_url): + error = msg.E_INIT_REPO_REFUSED.format( + template=core_msg.display(detail.name) + ) + print_error(escape(error)) + return InitResult(status="error", message=error) + if not lifecycle.valid_name(detail.name): + error = msg.E_INIT_NAME_REFUSED.format( + template=core_msg.display(template_slug or detail.name) + ) + print_error(escape(error)) + return InitResult(status="error", message=error) + # ── Resolve target directory ─────────────────────────────────────────── dest = Path(target_dir) if target_dir else Path(f"./{detail.name}") dest = dest.resolve() - # ── Confirm ──────────────────────────────────────────────────────────── - self._print_template_summary(detail) - console.print(f"[dim]Destination:[/dim] {dest}") - console.print() - - if not _agentic and not self.confirm("Clone this template?", default=True): - return InitResult(status="cancelled", message="Clone cancelled") - - # ── Clone ────────────────────────────────────────────────────────────── - github_url = detail.links.get("github") - if not github_url: - return InitResult(status="error", message="Template has no GitHub URL") + # ── Rerun contract: resume in a clone dg init made ───────────────────── + # Only dg's own record (in its state directory) says a directory is a + # clone dg made; nothing inside the directory is trusted for that, + # and no git command runs in it until the record does. + record: clones.CloneRecord | None = None + if lifecycle.directory_in_use(dest): + record = clones.load(os.path.realpath(dest)) + if record is None: + return self._dir_error(msg.E_INIT_DIR_IN_USE, detail, dest) + if record.template != detail.name: + return self._dir_error( + msg.E_INIT_DIR_OTHER_TEMPLATE, detail, dest, record.template + ) + # The same directory (not one made again at this path), owned by + # this user, before git runs in it; then git checks HEAD. + if ( + record.url != github_url + or not lifecycle.same_directory(dest, record.dev, record.ino) + or not lifecycle.clone_matches(dest, record.commit) + ): + return self._dir_error(msg.E_INIT_DIR_IN_USE, detail, dest) + phases: list[str] = list(record.phases) if record else [] + commit = record.commit if record else None + # The directory dg cloned into or resumes in, checked again right + # before anything is written or run in it. + identity = (record.dev, record.ino) if record else None + person = _person_at_terminal() + listed_steps = _commands(detail.config.install if detail.config else None) + + # An explicit --install approves setup, except under an AI agent, + # where only --yes does. Setup commands the template service lists + # are confirmed before anything is written. + approved = yes or (do_install and not _hard_signal()) + if record is None and listed_steps and not (person or approved or no_install): + return self._confirm_steps(detail, dest, listed_steps, cloned=False) + + if record is None: + # ── Confirm ──────────────────────────────────────────────────────── + self._print_template_summary(detail) + console.print(f"[dim]Destination:[/dim] {dest}") + console.print() - console.print() - with console.status("[blue]Cloning repository...[/blue]"): - try: - lifecycle.clone_template(github_url, dest) - except RuntimeError as e: - return InitResult(status="error", message=str(e)) + if person and not self.confirm("Clone this template?", default=True): + return InitResult(status="cancelled", message="Clone cancelled") - console.print(f"[green]✓[/green] Cloned to [cyan]{dest}[/cyan]") - console.print( - "[dim]The cloned directory is a full git repository — " - "you can add your own remote and push.[/dim]" - ) + # ── Clone ────────────────────────────────────────────────────────── + console.print() + with console.status("[blue]Cloning repository...[/blue]"): + try: + lifecycle.clone_template(github_url, dest) + except RuntimeError as e: + return InitResult(status="error", message=str(e)) + phases = ["clone"] + identity = lifecycle.directory_identity(dest) + commit = lifecycle.head_commit(dest) + self._save_progress(dest, detail.name, github_url, commit, phases, identity) + + console.print(f"[green]✓[/green] Cloned to [cyan]{dest}[/cyan]") + console.print( + "[dim]The cloned directory is a full git repository — " + "you can add your own remote and push.[/dim]" + ) + else: + console.print(f"[dim]Resuming in[/dim] [cyan]{dest}[/cyan]") # ── Inject API key ───────────────────────────────────────────────────── + if not _still_same(dest, identity): + return self._dir_error(msg.E_INIT_DIR_CHANGED, detail, dest) try: api_key = auth_manager.get_api_key() if api_key: @@ -214,12 +368,17 @@ def handle( ) # ── Setup (make check-prereqs && make init) ──────────────────────────── - installed = False + installed = "setup" in phases has_makefile = (dest / "Makefile").exists() - if not no_install: - if _agentic or do_install: + if not installed and not no_install: + steps = _MAKE_STEPS if has_makefile else listed_steps + if not steps: + run_setup = False + elif approved or (person and do_install): run_setup = True + elif not person: + return self._confirm_steps(detail, dest, steps, cloned=True) elif has_makefile: console.print() run_setup = self.confirm( @@ -229,6 +388,8 @@ def handle( run_setup = False if run_setup: + if not _still_same(dest, identity): + return self._dir_error(msg.E_INIT_DIR_CHANGED, detail, dest) console.print() if has_makefile: installed = lifecycle.run_make_init(dest) @@ -237,17 +398,27 @@ def handle( installed = lifecycle.run_lifecycle_step( detail.config.install, dest ) + if installed: + phases.append("setup") + self._save_progress( + dest, detail.name, github_url, commit, phases, identity + ) # ── Start step (optional) ────────────────────────────────────────────── + # Never started unasked: --start, or a person's answer. started = False if installed and detail.config and detail.config.start and not no_start: - if _agentic or do_start: + if do_start: run_start = True - else: + elif person: console.print() run_start = self.confirm("Run start step?", default=False) + else: + return self._confirm_start(detail, dest) if run_start: + if not _still_same(dest, identity): + return self._dir_error(msg.E_INIT_DIR_CHANGED, detail, dest) console.print() started = lifecycle.run_lifecycle_step(detail.config.start, dest) @@ -273,6 +444,126 @@ def handle( started=started, ) + # ── Confirmations and the rerun contract ────────────────────────────────── + + def _save_progress( + self, + dest: Path, + template: str, + url: str, + commit: str | None, + phases: list[str], + identity: tuple[int, int] | None, + ) -> None: + """Record the clone and what is done in dg's state, so a rerun + resumes. Without a commit there is nothing a rerun could check.""" + if commit is None or identity is None: + return + record = clones.CloneRecord( + os.path.realpath(dest), template, url, commit, phases, *identity + ) + try: + clones.save(record) + except OSError as exc: + text = msg.W_INIT_PROGRESS_UNSAVED.format( + exc=exc, prog=_prog_text(), dest=core_msg.display_path(dest) + ) + console.print(f"[yellow]{escape(text)}[/yellow]") + + def _confirm_steps( + self, detail: TemplateDetail, dest: Path, steps: list[str], *, cloned: bool + ) -> InitCodedError: + """Setup commands no one here can approve: confirmation_required.""" + next_step = _rerun(detail.name, dest, "--no-install", "--no-start") + shown = core_msg.display_path(dest) + if not _hard_signal(): + hint = ( + msg.H_CONFIRM_INIT_STEPS + if cloned + else msg.H_CONFIRM_INIT_STEPS_BEFORE_CLONE + ) + elif cloned: + hint = ( + msg.A_CONFIRM_INIT_STEPS if next_step else msg.A_CONFIRM_INIT_STEPS_NULL + ) + else: + hint = ( + msg.A_CONFIRM_INIT_STEPS_BEFORE_CLONE + if next_step + else msg.A_CONFIRM_INIT_STEPS_BEFORE_CLONE_NULL + ) + # Before the clone, the repository's Makefile is unknown, and if it + # has one its targets run instead of the listed commands; say so. + template = core_msg.display(detail.name) + listed = "; ".join(core_msg.display(step) for step in steps) + if cloned: + error = msg.E_CONFIRM_INIT_STEPS.format(template=template, steps=listed) + else: + error = msg.E_CONFIRM_INIT_STEPS_BEFORE_CLONE.format( + template=template, steps=listed + ) + return _coded( + error, + msg.CODE_CONFIRMATION_REQUIRED, + hint.format(dest=shown, prog=_prog_text()), + next_step, + {"flag": msg.YES_FLAG}, + ) + + def _confirm_start(self, detail: TemplateDetail, dest: Path) -> InitCodedError: + """A start step no one here asked for: confirmation_required.""" + next_step = _rerun(detail.name, dest, "--no-start") + if not _hard_signal(): + hint = msg.H_CONFIRM_INIT_START + else: + hint = ( + msg.A_CONFIRM_INIT_START if next_step else msg.A_CONFIRM_INIT_START_NULL + ) + error = msg.E_CONFIRM_INIT_START.format(template=core_msg.display(detail.name)) + return _coded( + error, + msg.CODE_CONFIRMATION_REQUIRED, + hint, + next_step, + {"flag": "--start"}, + ) + + def _dir_error( + self, + message: str, + detail: TemplateDetail, + dest: Path, + other: str | None = None, + ) -> InitCodedError: + """A directory dg init can't clone into or resume in.""" + from deepctl_core.transport import prog + + program = prog() + error = message.format( + dest=core_msg.display_path(dest), + template=core_msg.display(detail.name), + other=core_msg.display(other or ""), + prog=_prog_text(), + ) + return _coded( + error, + msg.CODE_INVALID_ARGUMENT, + msg.H_INIT_DIR, + f"{program} init --help" if program else None, + ) + + def _pick_needs_name(self) -> InitCodedError: + """No template named where the picker can't be shown.""" + from deepctl_core.transport import prog + + program = prog() + return _coded( + msg.E_CONFIRM_INIT_PICK.format(prog=_prog_text()), + msg.CODE_INVALID_ARGUMENT, + msg.H_CONFIRM_INIT_PICK.format(prog=_prog_text()), + f"{program} init --list" if program else None, + ) + def output_result(self, result: Any, config: Config) -> None: """Only emit structured output when an explicit --output format is set.""" from deepctl_core.output import get_output_format diff --git a/packages/deepctl-cmd-init/src/deepctl_cmd_init/lifecycle.py b/packages/deepctl-cmd-init/src/deepctl_cmd_init/lifecycle.py index 47afb667..fb806c53 100644 --- a/packages/deepctl-cmd-init/src/deepctl_cmd_init/lifecycle.py +++ b/packages/deepctl-cmd-init/src/deepctl_cmd_init/lifecycle.py @@ -3,9 +3,12 @@ from __future__ import annotations import os +import re import shutil +import stat import subprocess from typing import TYPE_CHECKING +from urllib.parse import urlsplit from deepctl_core.output import print_hint, print_info, print_warning from rich.console import Console @@ -58,8 +61,109 @@ def check_prereqs() -> list[tuple[str, str, str]]: return missing +# ── Running git ─────────────────────────────────────────────────────────────── + +# Settings every git command dg runs gets on its command line, so that +# configuration inside a repository can't run a program: no fsmonitor +# daemon command, no hooks, no pager, no external diff. (The commands dg +# runs in a repository, rev-parse, ls-files and check-ignore, apply no +# filters and open no connection.) +_NULL_DEVICE = "NUL" if os.name == "nt" else "/dev/null" +GIT_SAFE_CONFIG = ( + "-c", + "core.fsmonitor=false", + "-c", + f"core.hooksPath={_NULL_DEVICE}", + "-c", + "core.pager=cat", + "-c", + "diff.external=", +) + + +def run_git( + *args: str, repo: Path | None = None, inherit_git_env: bool = False +) -> subprocess.CompletedProcess[str] | None: + """Run ``git [-C repo] args`` with GIT_SAFE_CONFIG; None if git can't run. + + Inherited GIT_* variables are dropped unless ``inherit_git_env`` (for a + clone, which works on no existing repository): GIT_DIR, GIT_WORK_TREE + or GIT_INDEX_FILE would point git at another repository. Call it with + ``repo`` only for a directory dg cloned itself. + """ + if shutil.which("git") is None: + return None + env = None + if not inherit_git_env: + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + argv = ["git", "--no-pager", *GIT_SAFE_CONFIG] + if repo is not None: + argv += ["-C", str(repo)] + try: + return subprocess.run([*argv, *args], capture_output=True, text=True, env=env) + except OSError: + return None + + # ── Git clone ───────────────────────────────────────────────────────────────── +#: The GitHub organizations dg clones templates from. +TEMPLATE_ORGS = ("deepgram", "deepgram-starters") + +# A repository or template name: ASCII letters, digits, ".", "_" and "-", +# not starting with "-" (git or a shell would read it as an option). +_NAME = re.compile(r"[A-Za-z0-9._-]+") + + +def valid_name(name: object) -> bool: + """Whether ``name`` is a usable template or repository name.""" + return ( + isinstance(name, str) + and _NAME.fullmatch(name) is not None + and not name.startswith("-") + and name not in (".", "..") + ) + + +def is_template_repo(url: str) -> bool: + """Whether ``url`` is an https GitHub repository of a Deepgram organization. + + The template service supplies the URL, so only + ``https://github.com/deepgram/`` and + ``https://github.com/deepgram-starters/`` are cloned: no other + host, scheme, port, user name, query or fragment, and the repository + name is plain (``valid_name``): no escapes, separators or option-like + leading ``-``. + """ + if not url.isprintable() or any(c.isspace() for c in url): + return False # urlsplit drops tabs and newlines; git wouldn't + try: + parts = urlsplit(url) + port = parts.port + except ValueError: + return False + if parts.scheme != "https" or parts.netloc != "github.com" or port is not None: + return False + if parts.query or parts.fragment or "\\" in url: + return False + segments = parts.path.split("/") + # "", org, repo (and an optional trailing "") + if len(segments) == 4 and segments[3] == "": + segments = segments[:3] + if len(segments) != 3 or segments[0] != "": + return False + org, repo = segments[1], segments[2] + return org in TEMPLATE_ORGS and valid_name(repo) + + +def directory_in_use(target_dir: Path) -> bool: + """Whether ``target_dir`` exists as anything but an empty directory.""" + if target_dir.is_symlink() or (target_dir.exists() and not target_dir.is_dir()): + return True + if not target_dir.exists(): + return False + return any(target_dir.iterdir()) + def clone_template(github_url: str, target_dir: Path) -> None: """Clone a template repository, keeping the .git directory intact. @@ -69,25 +173,112 @@ def clone_template(github_url: str, target_dir: Path) -> None: Args: github_url: Full GitHub URL to clone. - target_dir: Local directory to clone into (must not exist). + target_dir: Local directory to clone into (must not exist, or be + an empty directory). Raises: - RuntimeError: If the directory already exists or git clone fails. + RuntimeError: If the directory is in use or git clone fails. """ - if target_dir.exists(): + if directory_in_use(target_dir): raise RuntimeError(f"Directory already exists: {target_dir}") - result = subprocess.run( - # "--" ends git's options: a server-supplied URL that starts with "-" - # is taken as the repository, never as a git option. - ["git", "clone", "--", github_url, str(target_dir)], - capture_output=True, - text=True, - ) + # "--" ends git's options: a server-supplied URL that starts with "-" + # (say --upload-pack=...) is taken as the repository, never an option. + result = run_git("clone", "--", github_url, str(target_dir), inherit_git_env=True) + if result is None: + raise RuntimeError("git clone failed: git could not be run") if result.returncode != 0: raise RuntimeError(f"git clone failed:\n{result.stderr.strip()}") +# ── The clone dg init made ──────────────────────────────────────────────────── + +_COMMIT = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") + + +def _git_out(target_dir: Path, *args: str) -> str | None: + """Output of a git command in ``target_dir``, or None if it failed.""" + result = run_git(*args, repo=target_dir) + if result is None or result.returncode != 0: + return None + return result.stdout.strip() or None + + +def head_commit(target_dir: Path) -> str | None: + """``HEAD`` of the repository dg just cloned into ``target_dir``.""" + commit = _git_out(target_dir, "rev-parse", "--verify", "HEAD") + return commit if commit and _COMMIT.fullmatch(commit) else None + + +_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0) + + +def same_directory(target_dir: Path, dev: int, ino: int) -> bool: + """Whether ``target_dir`` is still the directory dg cloned into. + + Checked before any git command runs in it. The directory, opened + without following a symbolic link, must have the recorded device and + inode (a directory deleted and made again at the same path has another + inode), and it and its ``.git`` must be owned by this user, with + ``.git`` a real directory. + + On Windows, where ``st_ino`` can be 0 or unstable and there are no + POSIX owners, only the path checks apply: no link, ``.git`` a real + directory. + """ + if os.name == "nt": + try: + top = os.lstat(target_dir) + git = os.lstat(target_dir / ".git") + except OSError: + return False + return stat.S_ISDIR(top.st_mode) and stat.S_ISDIR(git.st_mode) + try: + fd = os.open(target_dir, os.O_RDONLY | _O_DIRECTORY | _NOFOLLOW) + except OSError: + return False + try: + st = os.fstat(fd) + git = os.stat(".git", dir_fd=fd, follow_symlinks=False) + except OSError: + return False + finally: + os.close(fd) + uid = os.geteuid() + return ( + stat.S_ISDIR(st.st_mode) + and stat.S_ISDIR(git.st_mode) + and (st.st_dev, st.st_ino) == (dev, ino) + and st.st_uid == uid + and git.st_uid == uid + ) + + +def directory_identity(target_dir: Path) -> tuple[int, int] | None: + """Device and inode of ``target_dir`` (not following a link), or None.""" + try: + st = os.lstat(target_dir) + except OSError: + return None + return (st.st_dev, st.st_ino) if stat.S_ISDIR(st.st_mode) else None + + +def clone_matches(target_dir: Path, commit: str) -> bool: + """Whether ``target_dir`` is still at ``commit``, the clone dg recorded. + + Only for a directory dg's own clone record names: git runs in it. Its + own ``.git`` directory (not a link, or a file pointing elsewhere) must + make it the top of a work tree, with ``HEAD`` at ``commit``. + """ + git_dir = target_dir / ".git" + if git_dir.is_symlink() or not git_dir.is_dir(): + return False + top = _git_out(target_dir, "rev-parse", "--show-toplevel") + if top is None or os.path.realpath(top) != os.path.realpath(target_dir): + return False + return head_commit(target_dir) == commit + + # ── .env injection ──────────────────────────────────────────────────────────── @@ -125,27 +316,16 @@ def _restrict_to_owner(path: Path) -> None: def _git(target_dir: Path, *args: str) -> int | None: - """Exit code of ``git -C target_dir args``, or None if git can't run.""" - if shutil.which("git") is None: - return None - try: - # Only this project's repository: an inherited GIT_DIR, GIT_WORK_TREE - # or GIT_INDEX_FILE would point git at another one. - env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} - result = subprocess.run( - ["git", "-C", str(target_dir), *args], - capture_output=True, - text=True, - env=env, - ) - except OSError: - return None - return result.returncode + """Exit code of a git command in ``target_dir``, or None if git can't run.""" + result = run_git(*args, repo=target_dir) + return None if result is None else result.returncode def _env_ignored(target_dir: Path) -> bool | None: """git's answer to "is .env ignored?", or None when git can't say.""" - rc = _git(target_dir, "check-ignore", "-q", ".env") + # No global excludes file: one that is a FIFO would block the check. + # The repository's own .gitignore is what keeps .env out of a commit. + rc = _git(target_dir, "-c", "core.excludesFile=", "check-ignore", "-q", ".env") if rc in (0, 1): return rc == 0 return None diff --git a/packages/deepctl-cmd-init/src/deepctl_cmd_init/messages.py b/packages/deepctl-cmd-init/src/deepctl_cmd_init/messages.py new file mode 100644 index 00000000..b69d01a5 --- /dev/null +++ b/packages/deepctl-cmd-init/src/deepctl_cmd_init/messages.py @@ -0,0 +1,87 @@ +"""Strings for ``dg init``'s coded errors. + +The confirmation strings are the catalogue's (``E-CONFIRM-INIT-*``). H is +for a person, A for an AI agent; ``*_NULL`` when ``next_step`` is null. +The ``*_BEFORE_CLONE`` hints are their variants for a confirmation raised +before anything was cloned. +""" + +CODE_CONFIRMATION_REQUIRED = "confirmation_required" +CODE_INVALID_ARGUMENT = "invalid_argument" +YES_FLAG = "--yes" + +E_CONFIRM_INIT_STEPS = 'Template "{template}" wants to run setup commands: {steps}' +H_CONFIRM_INIT_STEPS = ( + "The template is cloned in {dest}. Review the commands, then run again with " + "--yes to run them, or --no-install to skip them." +) +A_CONFIRM_INIT_STEPS = ( + "The template is cloned in {dest}. next_step finishes without running the " + "setup commands. Show the user the commands and run them (with --yes) only " + "if the user approves." +) +A_CONFIRM_INIT_STEPS_NULL = ( + "The template is cloned in {dest}. Show the user the setup commands and ask " + "them to finish setup in their terminal." +) +# Before the clone, the setup that --yes approves is the listed commands, +# or the Makefile targets if the repository turns out to have a Makefile. +E_CONFIRM_INIT_STEPS_BEFORE_CLONE = ( + 'Template "{template}" has setup commands. --yes runs {steps}, or make ' + "check-prereqs and make init if the repository has a Makefile." +) +H_CONFIRM_INIT_STEPS_BEFORE_CLONE = ( + "Nothing was cloned. Review the commands, then run again with --yes to run " + "them, or --no-install to skip them." +) +A_CONFIRM_INIT_STEPS_BEFORE_CLONE = ( + "Nothing was cloned. next_step clones the template without running the " + "setup commands. Show the user the commands and run them (with --yes) only " + "if the user approves." +) +A_CONFIRM_INIT_STEPS_BEFORE_CLONE_NULL = ( + "Nothing was cloned. Show the user the setup commands and ask them to run " + "{prog} init in their terminal." +) + +E_CONFIRM_INIT_START = 'Template "{template}" can start the app after setup.' +H_CONFIRM_INIT_START = "Run again with --start to start it, or --no-start to skip." +A_CONFIRM_INIT_START = ( + "next_step skips starting the app. Start it only if the user asks." +) +A_CONFIRM_INIT_START_NULL = ( + "Ask the user whether to start the app; rerun with --no-start to skip it." +) + +# No template named, and no picker to choose one (an AI agent session). +E_CONFIRM_INIT_PICK = "{prog} init needs a template name when it can't show the picker." +H_CONFIRM_INIT_PICK = ( + "List templates with {prog} init --list, then run {prog} init